Skip to content

Commit 3fad1ee

Browse files
davidkyledavidkyle
authored
Un-assign persistent tasks as nodes exit the cluster (#37656)
PersistentTasksClusterService decides if a task should be reassigned by checking there is a node in the cluster with the same Id. If a node is restarted PersistentTasksClusterService may not observe the change and decide the task still has a valid assignment because the node's ephemeral Id is not used in that decision. This change un-assigns tasks as the nodes in the cluster change.
1 parent 2286118 commit 3fad1ee

File tree

5 files changed

+138
-2
lines changed

5 files changed

+138
-2
lines changed

‎server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.cluster.node.DiscoveryNodes;
3131
import org.elasticsearch.cluster.routing.allocation.AllocationService;
3232
import org.elasticsearch.discovery.DiscoverySettings;
33+
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
3334

3435
import java.util.ArrayList;
3536
import java.util.Collection;
@@ -187,6 +188,7 @@ protected ClusterState.Builder becomeMasterAndTrimConflictingNodes(ClusterState
187188
.blocks(currentState.blocks())
188189
.removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build();
189190
logger.trace("becomeMasterAndTrimConflictingNodes: {}", tmpState.nodes());
191+
tmpState = PersistentTasksCustomMetaData.deassociateDeadNodes(tmpState);
190192
return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false, "removed dead nodes on election"));
191193
}
192194

‎server/src/main/java/org/elasticsearch/cluster/coordination/NodeRemovalClusterStateTaskExecutor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.cluster.node.DiscoveryNode;
2727
import org.elasticsearch.cluster.node.DiscoveryNodes;
2828
import org.elasticsearch.cluster.routing.allocation.AllocationService;
29+
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
2930

3031
import java.util.List;
3132

@@ -91,8 +92,9 @@ public ClusterTasksResult<Task> execute(final ClusterState currentState, final L
9192

9293
protected ClusterTasksResult<Task> getTaskClusterTasksResult(ClusterState currentState, List<Task> tasks,
9394
ClusterState remainingNodesClusterState) {
95+
ClusterState ptasksDeassociatedState = PersistentTasksCustomMetaData.deassociateDeadNodes(remainingNodesClusterState);
9496
final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
95-
return resultBuilder.build(allocationService.deassociateDeadNodes(remainingNodesClusterState, true, describeTasks(tasks)));
97+
return resultBuilder.build(allocationService.deassociateDeadNodes(ptasksDeassociatedState, true, describeTasks(tasks)));
9698
}
9799

98100
// visible for testing

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,7 @@ private void deassociateDeadNodes(RoutingAllocation allocation) {
425425
for (ShardRouting shardRouting : node.copyShards()) {
426426
final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
427427
boolean delayed = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).nanos() > 0;
428-
UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]",
428+
UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left [" + node.nodeId() + "]",
429429
null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), delayed, AllocationStatus.NO_ATTEMPT);
430430
allocation.routingNodes().failShard(logger, shardRouting, unassignedInfo, indexMetaData, allocation.changes());
431431
}

‎server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetaData.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public final class PersistentTasksCustomMetaData extends AbstractNamedDiffable<M
6262

6363
public static final String TYPE = "persistent_tasks";
6464
private static final String API_CONTEXT = MetaData.XContentContext.API.toString();
65+
static final Assignment LOST_NODE_ASSIGNMENT = new Assignment(null, "awaiting reassignment after node loss");
6566

6667
// TODO: Implement custom Diff for tasks
6768
private final Map<String, PersistentTask<?>> tasks;
@@ -119,6 +120,11 @@ public PersistentTasksCustomMetaData(long lastAllocationId, Map<String, Persiste
119120
new ParseField("allocation_id_on_last_status_update"));
120121
}
121122

123+
124+
public static PersistentTasksCustomMetaData getPersistentTasksCustomMetaData(ClusterState clusterState) {
125+
return clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE);
126+
}
127+
122128
/**
123129
* Private builder used in XContent parser to build task-specific portion (params and state)
124130
*/
@@ -209,6 +215,39 @@ public static <Params extends PersistentTaskParams> PersistentTask<Params> getTa
209215
return null;
210216
}
211217

218+
/**
219+
* Unassign any persistent tasks executing on nodes that are no longer in
220+
* the cluster. If the task's assigment has a non-null executor node and that
221+
* node is no longer in the cluster then the assignment is set to
222+
* {@link #LOST_NODE_ASSIGNMENT}
223+
*
224+
* @param clusterState The clusterstate
225+
* @return If no changes the argument {@code clusterState} is returned else
226+
* a copy with the modified tasks
227+
*/
228+
public static ClusterState deassociateDeadNodes(ClusterState clusterState) {
229+
PersistentTasksCustomMetaData tasks = getPersistentTasksCustomMetaData(clusterState);
230+
if (tasks == null) {
231+
return clusterState;
232+
}
233+
234+
PersistentTasksCustomMetaData.Builder taskBuilder = PersistentTasksCustomMetaData.builder(tasks);
235+
for (PersistentTask<?> task : tasks.tasks()) {
236+
if (task.getAssignment().getExecutorNode() != null &&
237+
clusterState.nodes().nodeExists(task.getAssignment().getExecutorNode()) == false) {
238+
taskBuilder.reassignTask(task.getId(), LOST_NODE_ASSIGNMENT);
239+
}
240+
}
241+
242+
if (taskBuilder.isChanged() == false) {
243+
return clusterState;
244+
}
245+
246+
MetaData.Builder metaDataBuilder = MetaData.builder(clusterState.metaData());
247+
metaDataBuilder.putCustom(TYPE, taskBuilder.build());
248+
return ClusterState.builder(clusterState).metaData(metaDataBuilder).build();
249+
}
250+
212251
public static class Assignment {
213252
@Nullable
214253
private final String executorNode;

‎server/src/test/java/org/elasticsearch/persistent/PersistentTasksCustomMetaDataTests.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,14 @@
2121
import org.elasticsearch.ResourceNotFoundException;
2222
import org.elasticsearch.Version;
2323
import org.elasticsearch.client.transport.TransportClient;
24+
import org.elasticsearch.cluster.ClusterName;
25+
import org.elasticsearch.cluster.ClusterState;
2426
import org.elasticsearch.cluster.Diff;
2527
import org.elasticsearch.cluster.NamedDiff;
2628
import org.elasticsearch.cluster.metadata.MetaData;
2729
import org.elasticsearch.cluster.metadata.MetaData.Custom;
30+
import org.elasticsearch.cluster.node.DiscoveryNode;
31+
import org.elasticsearch.cluster.node.DiscoveryNodes;
2832
import org.elasticsearch.common.ParseField;
2933
import org.elasticsearch.common.UUIDs;
3034
import org.elasticsearch.common.bytes.BytesReference;
@@ -33,9 +37,11 @@
3337
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
3438
import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
3539
import org.elasticsearch.common.io.stream.StreamInput;
40+
import org.elasticsearch.common.io.stream.StreamOutput;
3641
import org.elasticsearch.common.io.stream.Writeable;
3742
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3843
import org.elasticsearch.common.xcontent.ToXContent;
44+
import org.elasticsearch.common.xcontent.XContentBuilder;
3945
import org.elasticsearch.common.xcontent.XContentFactory;
4046
import org.elasticsearch.common.xcontent.XContentParser;
4147
import org.elasticsearch.common.xcontent.XContentType;
@@ -65,6 +71,8 @@
6571
import static org.elasticsearch.test.VersionUtils.getPreviousVersion;
6672
import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
6773
import static org.hamcrest.Matchers.equalTo;
74+
import static org.hamcrest.Matchers.not;
75+
import static org.hamcrest.Matchers.sameInstance;
6876

6977
public class PersistentTasksCustomMetaDataTests extends AbstractDiffableSerializationTestCase<Custom> {
7078

@@ -307,6 +315,91 @@ public void testFeatureSerialization() throws IOException {
307315
assertThat(read.taskMap().keySet(), equalTo(Collections.singleton("test_compatible")));
308316
}
309317

318+
public void testDisassociateDeadNodes_givenNoPersistentTasks() {
319+
ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests")).build();
320+
ClusterState returnedState = PersistentTasksCustomMetaData.deassociateDeadNodes(originalState);
321+
assertThat(originalState, sameInstance(returnedState));
322+
}
323+
324+
public void testDisassociateDeadNodes_givenAssignedPersistentTask() {
325+
DiscoveryNodes nodes = DiscoveryNodes.builder()
326+
.add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
327+
.localNodeId("node1")
328+
.masterNodeId("node1")
329+
.build();
330+
331+
String taskName = "test/task";
332+
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder()
333+
.addTask("task-id", taskName, emptyTaskParams(taskName),
334+
new PersistentTasksCustomMetaData.Assignment("node1", "test assignment"));
335+
336+
ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests"))
337+
.nodes(nodes)
338+
.metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()))
339+
.build();
340+
ClusterState returnedState = PersistentTasksCustomMetaData.deassociateDeadNodes(originalState);
341+
assertThat(originalState, sameInstance(returnedState));
342+
343+
PersistentTasksCustomMetaData originalTasks = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(originalState);
344+
PersistentTasksCustomMetaData returnedTasks = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(returnedState);
345+
assertEquals(originalTasks, returnedTasks);
346+
}
347+
348+
public void testDisassociateDeadNodes() {
349+
DiscoveryNodes nodes = DiscoveryNodes.builder()
350+
.add(new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT))
351+
.localNodeId("node1")
352+
.masterNodeId("node1")
353+
.build();
354+
355+
String taskName = "test/task";
356+
PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder()
357+
.addTask("assigned-task", taskName, emptyTaskParams(taskName),
358+
new PersistentTasksCustomMetaData.Assignment("node1", "test assignment"))
359+
.addTask("task-on-deceased-node", taskName, emptyTaskParams(taskName),
360+
new PersistentTasksCustomMetaData.Assignment("left-the-cluster", "test assignment"));
361+
362+
ClusterState originalState = ClusterState.builder(new ClusterName("persistent-tasks-tests"))
363+
.nodes(nodes)
364+
.metaData(MetaData.builder().putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()))
365+
.build();
366+
ClusterState returnedState = PersistentTasksCustomMetaData.deassociateDeadNodes(originalState);
367+
assertThat(originalState, not(sameInstance(returnedState)));
368+
369+
PersistentTasksCustomMetaData originalTasks = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(originalState);
370+
PersistentTasksCustomMetaData returnedTasks = PersistentTasksCustomMetaData.getPersistentTasksCustomMetaData(returnedState);
371+
assertNotEquals(originalTasks, returnedTasks);
372+
373+
assertEquals(originalTasks.getTask("assigned-task"), returnedTasks.getTask("assigned-task"));
374+
assertNotEquals(originalTasks.getTask("task-on-deceased-node"), returnedTasks.getTask("task-on-deceased-node"));
375+
assertEquals(PersistentTasksCustomMetaData.LOST_NODE_ASSIGNMENT, returnedTasks.getTask("task-on-deceased-node").getAssignment());
376+
}
377+
378+
private PersistentTaskParams emptyTaskParams(String taskName) {
379+
return new PersistentTaskParams() {
380+
381+
@Override
382+
public XContentBuilder toXContent(XContentBuilder builder, Params params) {
383+
return builder;
384+
}
385+
386+
@Override
387+
public void writeTo(StreamOutput out) {
388+
389+
}
390+
391+
@Override
392+
public String getWriteableName() {
393+
return taskName;
394+
}
395+
396+
@Override
397+
public Version getMinimalSupportedVersion() {
398+
return Version.CURRENT;
399+
}
400+
};
401+
}
402+
310403
private Assignment randomAssignment() {
311404
if (randomBoolean()) {
312405
if (randomBoolean()) {

0 commit comments

Comments
 (0)