From 50696abbc9ab46cf9e4f7ef99ca46ec26887d24f Mon Sep 17 00:00:00 2001 From: Shailesh Gupta Date: Tue, 22 Mar 2022 22:51:40 +0530 Subject: [PATCH 1/4] YARN-11093: fix for reading files in timeline server in chronological order in case of fs-support-append to false --- .../hadoop/yarn/conf/YarnConfiguration.java | 4 +++- .../timeline/EntityGroupFSTimelineStore.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e61b5416e22a4..69f0e9dc00bbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3275,7 +3275,9 @@ public static boolean isAclEnabled(Configuration conf) { public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND = TIMELINE_SERVICE_PREFIX + "entity-file.fs-support-append"; - + public static final boolean TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND_DEFAULT + = true; + public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "with-user-dir"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 1f4a9f42a9f8c..d78f27a865b52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -76,9 +76,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -141,6 +143,7 @@ public class EntityGroupFSTimelineStore extends CompositeService private long unknownActiveMillis; private int appCacheMaxSize = 0; private boolean recoveryEnabled; + private boolean isAppendSupported; private Path checkpointFile; private ConcurrentMap> recoveredLogs = new ConcurrentHashMap>(); @@ -223,6 +226,10 @@ protected boolean removeEldestEntry( YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED, YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RECOVERY_ENABLED_DEFAULT); + isAppendSupported = conf.getBoolean( + YarnConfiguration.TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND, + YarnConfiguration.TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND_DEFAULT); + aclsEnabled = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); CallerContext.setCurrent( @@ -842,11 +849,28 @@ long scanForLogs() throws IOException { } String attemptDirName = statAttempt.getPath().getName(); RemoteIterator iterCache = list(statAttempt.getPath()); + List files = new ArrayList<>(); while (iterCache.hasNext()) { FileStatus statCache = iterCache.next(); if (!statCache.isFile()) { continue; } + files.add(statCache); + } + if (isAppendSupported) { + Collections.sort(files, new Comparator() { + @Override + public int compare(FileStatus o1, FileStatus o2) { + String[] ts1 = o1.getPath().getName().split("_"); + String[] ts2 = o2.getPath().getName().split("_"); + return (Integer.parseInt(ts1[ts1.length - 1]) - Integer.parseInt(ts2[ts2.length - 1])); + } + }); + } + Iterator fileIterator = files.iterator(); + + while (fileIterator.hasNext()) { + FileStatus statCache = fileIterator.next(); String filename = statCache.getPath().getName(); String owner = statCache.getOwner(); //YARN-10884:Owner of File is set to Null on WASB Append Operation.ATS fails to read such From e92416e0251fbdc21bbf82950701c14cb743a042 Mon Sep 17 00:00:00 2001 From: Shailesh Gupta Date: Wed, 23 Mar 2022 12:19:41 +0530 Subject: [PATCH 2/4] test cases fixed --- .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../yarn/server/timeline/EntityGroupFSTimelineStore.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 69f0e9dc00bbe..2347db5095b9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3277,7 +3277,7 @@ public static boolean isAclEnabled(Configuration conf) { + "entity-file.fs-support-append"; public static final boolean TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND_DEFAULT = true; - + public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "with-user-dir"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index d78f27a865b52..e0c90bb16f3f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -863,7 +863,11 @@ long scanForLogs() throws IOException { public int compare(FileStatus o1, FileStatus o2) { String[] ts1 = o1.getPath().getName().split("_"); String[] ts2 = o2.getPath().getName().split("_"); - return (Integer.parseInt(ts1[ts1.length - 1]) - Integer.parseInt(ts2[ts2.length - 1])); + if (StringUtils.isNumeric(ts1[ts1.length - 1]) && StringUtils.isNumeric(ts2[ts2.length - 1])) { + return (Integer.parseInt(ts1[ts1.length - 1]) - Integer.parseInt(ts2[ts2.length - 1])); + } else { + return o1.getPath().getName().compareTo(o2.getPath().getName()); + } } }); } From a5c1e3cd2e100f8d6c1035719cde558387232d78 Mon Sep 17 00:00:00 2001 From: Shailesh Gupta Date: Wed, 1 Mar 2023 22:44:19 +0530 Subject: [PATCH 3/4] checkstyle comments --- .../yarn/server/timeline/EntityGroupFSTimelineStore.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index e0c90bb16f3f6..95fa690db4c16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -863,8 +863,10 @@ long scanForLogs() throws IOException { public int compare(FileStatus o1, FileStatus o2) { String[] ts1 = o1.getPath().getName().split("_"); String[] ts2 = o2.getPath().getName().split("_"); - if (StringUtils.isNumeric(ts1[ts1.length - 1]) && StringUtils.isNumeric(ts2[ts2.length - 1])) { - return (Integer.parseInt(ts1[ts1.length - 1]) - Integer.parseInt(ts2[ts2.length - 1])); + if (StringUtils.isNumeric(ts1[ts1.length - 1]) + && StringUtils.isNumeric(ts2[ts2.length - 1])) { + return (Integer.parseInt(ts1[ts1.length - 1]) + - Integer.parseInt(ts2[ts2.length - 1])); } else { return o1.getPath().getName().compareTo(o2.getPath().getName()); } From 4b8706d600f805c33579d52c31fc81d471536a75 Mon Sep 17 00:00:00 2001 From: Shailesh Gupta Date: Thu, 2 Mar 2023 10:44:29 +0000 Subject: [PATCH 4/4] updated package-info --- .../org/apache/hadoop/yarn/server/timeline/package-info.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java index bf0fe79c3190e..b6e8161cdb7cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/package-info.java @@ -15,6 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/** + * This package provides ATS functionality. + */ @InterfaceAudience.Private package org.apache.hadoop.yarn.server.timeline; import org.apache.hadoop.classification.InterfaceAudience;