From bcfd17d92ce282e5a28c7595a32ea36e2f4864b0 Mon Sep 17 00:00:00 2001 From: Ferenc Erdelyi <55103964+ferdelyi@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:41:48 +0100 Subject: [PATCH] YARN-11639. CME and NPE in PriorityUtilizationQueueOrderingPolicy (#6455) --- .../policy/PriorityUtilizationQueueOrderingPolicy.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java index 995c2ea6b36bc..90ea082426d25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java @@ -28,6 +28,7 @@ .CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; +import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -254,9 +255,9 @@ public Iterator getAssignmentIterator(String partition) { // partitionToLookAt is a thread local variable, therefore it is safe to mutate it. PriorityUtilizationQueueOrderingPolicy.partitionToLookAt.set(partition); - // Sort the snapshot of the queues in order to avoid breaking the prerequisites of TimSort. - // See YARN-10178 for details. - return queues.stream().map(PriorityQueueResourcesForSorting::new).sorted( + // Copy (for thread safety) and sort the snapshot of the queues in order to avoid breaking + // the prerequisites of TimSort. See YARN-10178 for details. + return new ArrayList<>(queues).stream().map(PriorityQueueResourcesForSorting::new).sorted( new PriorityQueueComparator()).map(PriorityQueueResourcesForSorting::getQueue).collect( Collectors.toList()).iterator(); }