From 472704bafbb3cd9953e620ce3eebab5193d3bb63 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 14 Sep 2021 15:20:05 -0400 Subject: [PATCH 1/3] [ML] prefer least allocated model when a new node is added to the cluster --- .../TrainedModelAllocationClusterService.java | 151 ++++++++++-------- .../xpack/ml/job/NodeLoadDetector.java | 20 ++- ...nedModelAllocationClusterServiceTests.java | 80 ++++++++++ 3 files changed, 182 insertions(+), 69 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java index a90aaeaac8fd8..416aa08c24ff2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java @@ -40,12 +40,14 @@ import org.elasticsearch.xpack.ml.job.NodeLoadDetector; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.stream.Collectors; public class TrainedModelAllocationClusterService implements ClusterStateListener { @@ -245,8 +247,7 @@ ClusterState createModelAllocation(ClusterState currentState, StartTrainedModelD Set shuttingDownNodes = nodesShuttingDown(currentState); Map nodeToReason = new TreeMap<>(); for (DiscoveryNode node : currentState.getNodes().getAllNodes()) { - if (StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node) - && shuttingDownNodes.contains(node.getId()) == false) { + if (StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node) && shuttingDownNodes.contains(node.getId()) == false) { Optional maybeError = nodeHasCapacity(currentState, params, node); if (maybeError.isPresent()) { nodeToReason.put(node.getName(), maybeError.get()); @@ -298,7 +299,7 @@ static ClusterState updateModelRoutingTable(ClusterState currentState, UpdateTra ) .collect(Collectors.toList()); final TrainedModelAllocation existingAllocation = metadata.getModelAllocation(modelId); - final TrainedModelAllocationMetadata.Builder builder = TrainedModelAllocationMetadata.builder(currentState); + final TrainedModelAllocationMetadata.Builder builder = TrainedModelAllocationMetadata.builder(currentState); // If state is stopped, this indicates the node process is closed, remove the node from the allocation if (request.getRoutingState().getState().equals(RoutingState.STOPPED)) { if (existingAllocation == null || existingAllocation.isRoutedToNode(nodeId) == false) { @@ -313,20 +314,20 @@ static ClusterState updateModelRoutingTable(ClusterState currentState, UpdateTra } // If we are stopping, don't update anything if (existingAllocation.getAllocationState().equals(AllocationState.STOPPING)) { - logger.debug(() -> new ParameterizedMessage( - "[{}] requested update from node [{}] to update route state to [{}]", - modelId, - nodeId, - request.getRoutingState() - )); + logger.debug( + () -> new ParameterizedMessage( + "[{}] requested update from node [{}] to update route state to [{}]", + modelId, + nodeId, + request.getRoutingState() + ) + ); return currentState; } if (existingAllocation.isRoutedToNode(nodeId) == false) { throw new ResourceNotFoundException("allocation for model with id [{}]] is not routed to node [{}]", modelId, nodeId); } - builder.getAllocation(modelId) - .updateExistingRoutingEntry(nodeId, request.getRoutingState()) - .calculateAndSetAllocationState(); + builder.getAllocation(modelId).updateExistingRoutingEntry(nodeId, request.getRoutingState()).calculateAndSetAllocationState(); return update(currentState, builder); } @@ -342,7 +343,7 @@ static ClusterState removeAllocation(ClusterState currentState, String modelId) static ClusterState removeAllAllocations(ClusterState currentState) { if (TrainedModelAllocationMetadata.fromState(currentState).modelAllocations().isEmpty()) { return currentState; - }; + } return ClusterState.builder(currentState) .metadata( Metadata.builder(currentState.metadata()) @@ -356,64 +357,63 @@ ClusterState addRemoveAllocationNodes(ClusterState currentState) { final TrainedModelAllocationMetadata previousState = TrainedModelAllocationMetadata.fromState(currentState); final TrainedModelAllocationMetadata.Builder builder = TrainedModelAllocationMetadata.builder(currentState); Set shuttingDownNodes = nodesShuttingDown(currentState); - Set currentNotShuttingDownNodes = currentState.getNodes() + Map currentNotShuttingDownNodes = currentState.getNodes() .getAllNodes() .stream() - .map(DiscoveryNode::getId) - .filter(id -> shuttingDownNodes.contains(id) == false) - .collect(Collectors.toSet()); - // TODO: make more efficient, right now this is O(nm) where n = sizeof(models) and m = sizeof(nodes) - // It could probably be O(max(n, m)) - // Add nodes and keep track of currently routed nodes - // Should we indicate a partial allocation somehow if some nodes don't have space? - for (Map.Entry modelAllocationEntry : previousState.modelAllocations().entrySet()) { - // Don't bother adding/removing nodes if this allocation is stopping - if (modelAllocationEntry.getValue().getAllocationState().equals(AllocationState.STOPPING)) { - continue; - } - final String modelId = modelAllocationEntry.getKey(); - Map nodeToReason = new TreeMap<>(); - for (DiscoveryNode node : currentState.getNodes()) { - // Only add the route if the node is NOT shutting down, this would be a weird case of the node - // just being added to the cluster and immediately shutting down... - if (shuttingDownNodes.contains(node.getId()) == false - && StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node) - && modelAllocationEntry.getValue().isRoutedToNode(node.getId()) == false) { - Optional failure = nodeHasCapacity(currentState, modelAllocationEntry.getValue().getTaskParams(), node); - if (failure.isPresent()) { - nodeToReason.put(node.getName(), failure.get()); - } else { - builder.getAllocation(modelId).addNewRoutingEntry(node.getId()); + .filter(node -> shuttingDownNodes.contains(node.getId()) == false) + .collect(Collectors.toMap(DiscoveryNode::getId, Function.identity())); + // TODO: make more efficient, we iterate every entry, sorting by nodes routed (fewest to most) + previousState.modelAllocations() + .entrySet() + .stream() + .filter(entry -> entry.getValue().getAllocationState().equals(AllocationState.STOPPING) == false) + .sorted(Comparator.comparing(e -> e.getValue().getNodeRoutingTable().size())) + .forEach(modelAllocationEntry -> { + final String modelId = modelAllocationEntry.getKey(); + Map nodeToReason = new TreeMap<>(); + for (DiscoveryNode node : currentNotShuttingDownNodes.values()) { + if (StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node) + && modelAllocationEntry.getValue().isRoutedToNode(node.getId()) == false) { + Optional failure = nodeHasCapacity( + currentState, + builder, + modelAllocationEntry.getValue().getTaskParams(), + node + ); + if (failure.isPresent()) { + nodeToReason.put(node.getName(), failure.get()); + } else { + builder.getAllocation(modelId).addNewRoutingEntry(node.getId()); + } } } - } - if (nodeToReason.isEmpty() == false) { - builder.getAllocation(modelId) - .setReason( - nodeToReason.entrySet() - .stream() - .map( - entry -> String.format( - Locale.ROOT, - "Not allocating on node [%s]. Reason: %s", - entry.getKey(), - entry.getValue() + if (nodeToReason.isEmpty() == false) { + builder.getAllocation(modelId) + .setReason( + nodeToReason.entrySet() + .stream() + .map( + entry -> String.format( + Locale.ROOT, + "Not allocating on node [%s]. Reason: %s", + entry.getKey(), + entry.getValue() + ) ) - ) - .collect(Collectors.joining("|")) - ); - } else { - builder.getAllocation(modelId).clearReason(); - } - for (String nodeId : modelAllocationEntry.getValue().getNodeRoutingTable().keySet()) { - if (currentNotShuttingDownNodes.contains(nodeId) == false) { - builder.getAllocation(modelId).removeRoutingEntry(nodeId); + .collect(Collectors.joining("|")) + ); + } else { + builder.getAllocation(modelId).clearReason(); } - } - // It may be we moved from STARTED to PARTIALLY_STARTED with the addition of new nodes - // Or moved from PARTIALLY_STARTED to STARTED if a node was removed - builder.getAllocation(modelId).calculateAndSetAllocationState(); - } + for (String nodeId : modelAllocationEntry.getValue().getNodeRoutingTable().keySet()) { + if (currentNotShuttingDownNodes.containsKey(nodeId) == false) { + builder.getAllocation(modelId).removeRoutingEntry(nodeId); + } + } + // It may be we moved from STARTED to PARTIALLY_STARTED with the addition of new nodes + // Or moved from PARTIALLY_STARTED to STARTED if a node was removed + builder.getAllocation(modelId).calculateAndSetAllocationState(); + }); return update(currentState, builder); } @@ -448,8 +448,24 @@ static boolean shouldAllocateModels(final ClusterChangedEvent event) { Optional nodeHasCapacity(ClusterState state, StartTrainedModelDeploymentAction.TaskParams params, DiscoveryNode node) { NodeLoad load = nodeLoadDetector.detectNodeLoad(state, true, node, Integer.MAX_VALUE, maxMemoryPercentage, useAuto); + return handleNodeLoad(load, node.getId(), params); + } + + Optional nodeHasCapacity( + ClusterState state, + TrainedModelAllocationMetadata.Builder builder, + StartTrainedModelDeploymentAction.TaskParams params, + DiscoveryNode node + ) { + NodeLoad load = builder.isChanged() + ? nodeLoadDetector.detectNodeLoad(state, builder.build(), true, node, Integer.MAX_VALUE, maxMemoryPercentage, useAuto) + : nodeLoadDetector.detectNodeLoad(state, true, node, Integer.MAX_VALUE, maxMemoryPercentage, useAuto); + return handleNodeLoad(load, node.getId(), params); + } + + Optional handleNodeLoad(NodeLoad load, String nodeId, StartTrainedModelDeploymentAction.TaskParams params) { if (Strings.isNullOrEmpty(load.getError()) == false) { - logger.warn("[{}] failed to calculate current node load with error [{}]", params.getModelId(), node.getId()); + logger.warn("[{}] failed to calculate current node load with error [{}]", params.getModelId(), nodeId); return Optional.of(load.getError()); } if (load.getFreeMemory() < params.estimateMemoryUsageBytes()) { @@ -464,8 +480,7 @@ Optional nodeHasCapacity(ClusterState state, StartTrainedModelDeployment load.getAssignedJobMemory(), ByteSizeValue.ofBytes(load.getAssignedJobMemory()).toString(), params.estimateMemoryUsageBytes(), - ByteSizeValue.ofBytes(params.estimateMemoryUsageBytes()).toString() - } + ByteSizeValue.ofBytes(params.estimateMemoryUsageBytes()).toString() } ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java index 8722370e67fa0..c019c5c3dfc7e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/NodeLoadDetector.java @@ -48,6 +48,24 @@ public NodeLoad detectNodeLoad(ClusterState clusterState, int dynamicMaxOpenJobs, int maxMachineMemoryPercent, boolean useAutoMachineMemoryCalculation) { + return detectNodeLoad( + clusterState, + TrainedModelAllocationMetadata.fromState(clusterState), + allNodesHaveDynamicMaxWorkers, + node, + dynamicMaxOpenJobs, + maxMachineMemoryPercent, + useAutoMachineMemoryCalculation + ); + } + + public NodeLoad detectNodeLoad(ClusterState clusterState, + TrainedModelAllocationMetadata allocationMetadata, + boolean allNodesHaveDynamicMaxWorkers, + DiscoveryNode node, + int dynamicMaxOpenJobs, + int maxMachineMemoryPercent, + boolean useAutoMachineMemoryCalculation) { PersistentTasksCustomMetadata persistentTasks = clusterState.getMetadata().custom(PersistentTasksCustomMetadata.TYPE); Map nodeAttributes = node.getAttributes(); List errors = new ArrayList<>(); @@ -80,7 +98,7 @@ public NodeLoad detectNodeLoad(ClusterState clusterState, return nodeLoad.setError(Strings.collectionToCommaDelimitedString(errors)).build(); } updateLoadGivenTasks(nodeLoad, persistentTasks); - updateLoadGivenModelAllocations(nodeLoad, TrainedModelAllocationMetadata.fromState(clusterState)); + updateLoadGivenModelAllocations(nodeLoad, allocationMetadata); return nodeLoad.build(); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java index 44e53c9e754f0..e332b3bcb6b71 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterServiceTests.java @@ -368,6 +368,86 @@ public void testAddRemoveAllocationNodes() { ); } + public void testAddRemoveAllocationNodesPrioritizesAllocationsWithFewerNodes() { + ClusterState currentState = ClusterState.builder(new ClusterName("testAddRemoveAllocationNodes")) + .nodes( + DiscoveryNodes.builder() + .add(buildNode("ml-node-with-room", true, ByteSizeValue.ofGb(4).getBytes())) + .add(buildNode("new-ml-node-with-just-enough-room", true, ByteSizeValue.ofGb(8).getBytes())) + .add(buildNode("ml-node-without-room", true, 1000L)) + .add(buildNode("not-ml-node", false, ByteSizeValue.ofGb(4).getBytes())) + .add(buildNode("ml-node-shutting-down", true, ByteSizeValue.ofGb(4).getBytes())) + .add(buildOldNode("old-versioned-ml-node-with-room", true, ByteSizeValue.ofGb(4).getBytes())) + .build() + ) + .metadata( + Metadata.builder() + .putCustom(NodesShutdownMetadata.TYPE, shutdownMetadata("ml-node-shutting-down")) + .putCustom( + TrainedModelAllocationMetadata.NAME, + TrainedModelAllocationMetadata.Builder.empty() + .addNewAllocation( + "model-1", + TrainedModelAllocation.Builder.empty(newParams("model-1", ByteSizeValue.ofGb(1).getBytes())) + .addNewRoutingEntry("ml-node-with-room") + .updateExistingRoutingEntry("ml-node-with-room", started()) + .addNewRoutingEntry("old-ml-node-with-room") + .updateExistingRoutingEntry("old-ml-node-with-room", started()) + .addNewRoutingEntry("ml-node-shutting-down") + ) + .addNewAllocation( + "model-2", + TrainedModelAllocation.Builder.empty(newParams("model-2", ByteSizeValue.ofGb(1).getBytes())) + .addNewRoutingEntry("ml-node-with-room") + ).addNewAllocation( + "model-3", + TrainedModelAllocation.Builder.empty(newParams("model-3", ByteSizeValue.ofGb(1).getBytes())) + ) + .build() + ) + ) + .build(); + TrainedModelAllocationClusterService trainedModelAllocationClusterService = createClusterService(); + + ClusterState modified = trainedModelAllocationClusterService.addRemoveAllocationNodes(currentState); + TrainedModelAllocationMetadata trainedModelAllocationMetadata = TrainedModelAllocationMetadata.fromState(modified); + assertThat(trainedModelAllocationMetadata.modelAllocations(), allOf(hasKey("model-1"), hasKey("model-2"), hasKey("model-3"))); + + assertThat(trainedModelAllocationMetadata.getModelAllocation("model-1").getNodeRoutingTable().keySet(), hasSize(1)); + assertThat( + trainedModelAllocationMetadata.getModelAllocation("model-1").getNodeRoutingTable(), + allOf(hasKey("ml-node-with-room")) + ); + assertNodeState(trainedModelAllocationMetadata, "model-1", "ml-node-with-room", RoutingState.STARTED); + assertThat( + trainedModelAllocationMetadata.modelAllocations().get("model-1").getAllocationState(), + equalTo(AllocationState.STARTED) + ); + + assertThat(trainedModelAllocationMetadata.getModelAllocation("model-2").getNodeRoutingTable().keySet(), hasSize(1)); + assertThat( + trainedModelAllocationMetadata.getModelAllocation("model-2").getNodeRoutingTable(), + allOf(hasKey("ml-node-with-room")) + ); + assertNodeState(trainedModelAllocationMetadata, "model-2", "ml-node-with-room", RoutingState.STARTING); + assertThat( + trainedModelAllocationMetadata.modelAllocations().get("model-2").getAllocationState(), + equalTo(AllocationState.STARTING) + ); + + assertThat(trainedModelAllocationMetadata.getModelAllocation("model-3").getNodeRoutingTable().keySet(), hasSize(1)); + assertThat( + trainedModelAllocationMetadata.getModelAllocation("model-3").getNodeRoutingTable(), + allOf(hasKey("new-ml-node-with-just-enough-room")) + ); + assertNodeState(trainedModelAllocationMetadata, "model-3", "new-ml-node-with-just-enough-room", RoutingState.STARTING); + assertThat( + trainedModelAllocationMetadata.modelAllocations().get("model-3").getAllocationState(), + equalTo(AllocationState.STARTING) + ); + } + + public void testShouldAllocateModels() { String model1 = "model-1"; String model2 = "model-2"; From 6406ebb8fcd71df4fb0e709af69d7abfb00f716f Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 15 Sep 2021 09:33:59 -0400 Subject: [PATCH 2/3] removing unused code --- .../allocation/TrainedModelAllocationClusterService.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java index 416aa08c24ff2..3a6ee0f09acaf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java @@ -41,7 +41,6 @@ import java.util.Collections; import java.util.Comparator; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -290,14 +289,6 @@ static ClusterState updateModelRoutingTable(ClusterState currentState, UpdateTra logger.trace( () -> new ParameterizedMessage("[{}] [{}] current metadata before update {}", modelId, nodeId, Strings.toString(metadata)) ); - Set shuttingDownNodes = nodesShuttingDown(currentState); - List allocatableNodes = currentState.nodes() - .getAllNodes() - .stream() - .filter( - d -> StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(d) && shuttingDownNodes.contains(d.getId()) == false - ) - .collect(Collectors.toList()); final TrainedModelAllocation existingAllocation = metadata.getModelAllocation(modelId); final TrainedModelAllocationMetadata.Builder builder = TrainedModelAllocationMetadata.builder(currentState); // If state is stopped, this indicates the node process is closed, remove the node from the allocation From 3bd697dc9d882529d8a6f791a409e95bb1f757b5 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 16 Sep 2021 07:54:55 -0400 Subject: [PATCH 3/3] addressing PR comments --- .../TrainedModelAllocationClusterService.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java index 3a6ee0f09acaf..9c0706ec53d4c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/allocation/TrainedModelAllocationClusterService.java @@ -348,10 +348,12 @@ ClusterState addRemoveAllocationNodes(ClusterState currentState) { final TrainedModelAllocationMetadata previousState = TrainedModelAllocationMetadata.fromState(currentState); final TrainedModelAllocationMetadata.Builder builder = TrainedModelAllocationMetadata.builder(currentState); Set shuttingDownNodes = nodesShuttingDown(currentState); - Map currentNotShuttingDownNodes = currentState.getNodes() + Map currentEligibleNodes = currentState.getNodes() .getAllNodes() .stream() - .filter(node -> shuttingDownNodes.contains(node.getId()) == false) + // TODO: Change when we update `mayAllocateToNode` + .filter(node -> shuttingDownNodes.contains(node.getId()) == false + && StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node)) .collect(Collectors.toMap(DiscoveryNode::getId, Function.identity())); // TODO: make more efficient, we iterate every entry, sorting by nodes routed (fewest to most) previousState.modelAllocations() @@ -362,15 +364,12 @@ ClusterState addRemoveAllocationNodes(ClusterState currentState) { .forEach(modelAllocationEntry -> { final String modelId = modelAllocationEntry.getKey(); Map nodeToReason = new TreeMap<>(); - for (DiscoveryNode node : currentNotShuttingDownNodes.values()) { - if (StartTrainedModelDeploymentAction.TaskParams.mayAllocateToNode(node) - && modelAllocationEntry.getValue().isRoutedToNode(node.getId()) == false) { - Optional failure = nodeHasCapacity( - currentState, - builder, - modelAllocationEntry.getValue().getTaskParams(), - node - ); + for (DiscoveryNode node : currentEligibleNodes.values()) { + if (modelAllocationEntry.getValue().isRoutedToNode(node.getId()) == false) { + Optional failure = builder.isChanged() ? + // We use the builder only if we have changed, there is no point in creating a new object if we haven't changed + nodeHasCapacity(currentState, builder, modelAllocationEntry.getValue().getTaskParams(), node) : + nodeHasCapacity(currentState, modelAllocationEntry.getValue().getTaskParams(), node); if (failure.isPresent()) { nodeToReason.put(node.getName(), failure.get()); } else { @@ -397,7 +396,7 @@ ClusterState addRemoveAllocationNodes(ClusterState currentState) { builder.getAllocation(modelId).clearReason(); } for (String nodeId : modelAllocationEntry.getValue().getNodeRoutingTable().keySet()) { - if (currentNotShuttingDownNodes.containsKey(nodeId) == false) { + if (currentEligibleNodes.containsKey(nodeId) == false) { builder.getAllocation(modelId).removeRoutingEntry(nodeId); } } @@ -442,15 +441,24 @@ Optional nodeHasCapacity(ClusterState state, StartTrainedModelDeployment return handleNodeLoad(load, node.getId(), params); } + /** + * Gather current node capacity taking the passed allocation metadata into account instead of the one stored in cluster state. + */ Optional nodeHasCapacity( ClusterState state, TrainedModelAllocationMetadata.Builder builder, StartTrainedModelDeploymentAction.TaskParams params, DiscoveryNode node ) { - NodeLoad load = builder.isChanged() - ? nodeLoadDetector.detectNodeLoad(state, builder.build(), true, node, Integer.MAX_VALUE, maxMemoryPercentage, useAuto) - : nodeLoadDetector.detectNodeLoad(state, true, node, Integer.MAX_VALUE, maxMemoryPercentage, useAuto); + NodeLoad load = nodeLoadDetector.detectNodeLoad( + state, + builder.build(), + true, + node, + Integer.MAX_VALUE, + maxMemoryPercentage, + useAuto + ); return handleNodeLoad(load, node.getId(), params); }