Node repurpose tool (#39403)

When a node is repurposed to master/no-data or no-master/no-data, v7.x
will not start (see #37748 and #37347). The `elasticsearch repurpose`
tool can fix this by cleaning up the problematic data.

Author: henningandersen

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -354,7 +354,7 @@ public void test92ElasticsearchNodeCliPackaging() {
         Platforms.PlatformAction action = () -> {
             final Result result = sh.run(bin.elasticsearchNode + " -h");
             assertThat(result.stdout,
-                    containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes"));
+                    containsString("A CLI tool to do unsafe cluster and index manipulations on current node"));
         };
 
         if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) {

server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.cluster.coordination;
 
-import joptsimple.OptionSet;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -47,24 +46,18 @@ public DetachClusterCommand() {
         super("Detaches this node from its cluster, allowing it to unsafely join a new cluster");
     }
 
-    @Override
-    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        super.execute(terminal, options, env);
-
-        processNodePathsWithLock(terminal, options, env);
-
-        terminal.println(NODE_DETACHED_MSG);
-    }
 
     @Override
-    protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException {
+    protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         final Tuple<Manifest, MetaData> manifestMetaDataTuple = loadMetaData(terminal, dataPaths);
         final Manifest manifest = manifestMetaDataTuple.v1();
         final MetaData metaData = manifestMetaDataTuple.v2();
 
         confirm(terminal, CONFIRMATION_MSG);
 
         writeNewMetaData(terminal, manifest, updateCurrentTerm(), metaData, updateMetaData(metaData), dataPaths);
+
+        terminal.println(NODE_DETACHED_MSG);
     }
 
     // package-private for tests

server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java

@@ -51,14 +51,14 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
                     "\n" +
                     "    WARNING: Elasticsearch MUST be stopped before running this tool." +
                     "\n";
-    static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
+    protected static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?";
     static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?";
     static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?";
-    static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " +
-            "bootstrapped?";
+    protected static final String GLOBAL_GENERATION_MISSING_MSG =
+        "no metadata is referenced from the manifest file, cluster has never been bootstrapped?";
     static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?";
     static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk";
-    static final String ABORTED_BY_USER_MSG = "aborted by user";
+    protected static final String ABORTED_BY_USER_MSG = "aborted by user";
     final OptionSpec<Integer> nodeOrdinalOption;
 
     public ElasticsearchNodeCommand(String description) {
@@ -80,7 +80,7 @@ protected void processNodePathsWithLock(Terminal terminal, OptionSet options, En
             if (dataPaths.length == 0) {
                 throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG);
             }
-            processNodePaths(terminal, dataPaths);
+            processNodePaths(terminal, dataPaths, env);
         } catch (LockObtainFailedException ex) {
             throw new ElasticsearchException(
                     FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]");
@@ -116,11 +116,31 @@ protected void confirm(Terminal terminal, String msg) {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+    protected final void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         terminal.println(STOP_WARNING_MSG);
+        if (validateBeforeLock(terminal, env)) {
+            processNodePathsWithLock(terminal, options, env);
+        }
+    }
+
+    /**
+     * Validate that the command can run before taking any locks.
+     * @param terminal the terminal to print to
+     * @param env the env to validate.
+     * @return true to continue, false to stop (must print message in validate).
+     */
+    protected boolean validateBeforeLock(Terminal terminal, Environment env) {
+        return true;
     }
 
-    protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException;
+
+    /**
+     * Process the paths. Locks for the paths is held during this method invocation.
+     * @param terminal the terminal to use for messages
+     * @param dataPaths the paths of the node to process
+     * @param env the env of the node to process
+     */
+    protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException;
 
 
     protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long newCurrentTerm,

server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.cli.CommandLoggingConfigurator;
 import org.elasticsearch.cli.MultiCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.NodeRepurposeCommand;
 
 // NodeToolCli does not extend LoggingAwareCommand, because LoggingAwareCommand performs logging initialization
 // after LoggingAwareCommand instance is constructed.
@@ -32,10 +33,12 @@
 public class NodeToolCli extends MultiCommand {
 
     public NodeToolCli() {
-        super("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes", ()->{});
+        super("A CLI tool to do unsafe cluster and index manipulations on current node",
+            ()->{});
         CommandLoggingConfigurator.configureLoggingWithoutConfig();
         subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand());
         subcommands.put("detach-cluster", new DetachClusterCommand());
+        subcommands.put("repurpose", new NodeRepurposeCommand());
     }
 
     public static void main(String[] args) throws Exception {

server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.cluster.coordination;
 
-import joptsimple.OptionSet;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -72,22 +71,18 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        super.execute(terminal, options, env);
-
+    protected boolean validateBeforeLock(Terminal terminal, Environment env) {
         Settings settings = env.settings();
         terminal.println(Terminal.Verbosity.VERBOSE, "Checking node.master setting");
         Boolean master = Node.NODE_MASTER_SETTING.get(settings);
         if (master == false) {
             throw new ElasticsearchException(NOT_MASTER_NODE_MSG);
         }
 
-        processNodePathsWithLock(terminal, options, env);
-
-        terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
+        return true;
     }
 
-    protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException {
+    protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
         final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
         if (nodeMetaData == null) {
@@ -130,5 +125,7 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx
                 .build();
 
         writeNewMetaData(terminal, manifest, manifest.getCurrentTerm(), metaData, newMetaData, dataPaths);
+
+        terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG);
     }
 }

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -128,7 +128,11 @@ public Path resolve(ShardId shardId) {
          * ${data.paths}/nodes/{node.id}/indices/{index.uuid}
          */
         public Path resolve(Index index) {
-            return indicesPath.resolve(index.getUUID());
+            return resolve(index.getUUID());
+        }
+
+        Path resolve(String uuid) {
+            return indicesPath.resolve(uuid);
         }
 
         @Override
@@ -1050,28 +1054,48 @@ private static void ensureAtomicMoveSupported(final NodePath[] nodePaths) throws
     }
 
     private void ensureNoShardData(final NodePath[] nodePaths) throws IOException {
-        List<Path> shardDataPaths = collectIndexSubPaths(nodePaths, this::isShardPath);
+        List<Path> shardDataPaths = collectShardDataPaths(nodePaths);
         if (shardDataPaths.isEmpty() == false) {
             throw new IllegalStateException("Node is started with "
                 + Node.NODE_DATA_SETTING.getKey()
                 + "=false, but has shard data: "
-                + shardDataPaths);
+                + shardDataPaths
+                + ". Use 'elasticsearch-node repurpose' tool to clean up"
+            );
         }
     }
 
     private void ensureNoIndexMetaData(final NodePath[] nodePaths) throws IOException {
-        List<Path> indexMetaDataPaths = collectIndexSubPaths(nodePaths, this::isIndexMetaDataPath);
+        List<Path> indexMetaDataPaths = collectIndexMetaDataPaths(nodePaths);
         if (indexMetaDataPaths.isEmpty() == false) {
             throw new IllegalStateException("Node is started with "
                 + Node.NODE_DATA_SETTING.getKey()
                 + "=false and "
                 + Node.NODE_MASTER_SETTING.getKey()
                 + "=false, but has index metadata: "
-                + indexMetaDataPaths);
+                + indexMetaDataPaths
+                + ". Use 'elasticsearch-node repurpose' tool to clean up"
+            );
         }
     }
 
-    private List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> subPathPredicate) throws IOException {
+    /**
+     * Collect the paths containing shard data in the indicated node paths. The returned paths will point to the shard data folder.
+     */
+    static List<Path> collectShardDataPaths(NodePath[] nodePaths) throws IOException {
+        return collectIndexSubPaths(nodePaths, NodeEnvironment::isShardPath);
+    }
+
+
+    /**
+     * Collect the paths containing index meta data in the indicated node paths. The returned paths will point to the
+     * {@link MetaDataStateFormat#STATE_DIR_NAME} folder
+     */
+    static List<Path> collectIndexMetaDataPaths(NodePath[] nodePaths) throws IOException {
+        return collectIndexSubPaths(nodePaths, NodeEnvironment::isIndexMetaDataPath);
+    }
+
+    private static List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> subPathPredicate) throws IOException {
         List<Path> indexSubPaths = new ArrayList<>();
         for (NodePath nodePath : nodePaths) {
             Path indicesPath = nodePath.indicesPath;
@@ -1093,12 +1117,12 @@ private List<Path> collectIndexSubPaths(NodePath[] nodePaths, Predicate<Path> su
         return indexSubPaths;
     }
 
-    private boolean isShardPath(Path path) {
+    private static boolean isShardPath(Path path) {
         return Files.isDirectory(path)
             && path.getFileName().toString().chars().allMatch(Character::isDigit);
     }
 
-    private boolean isIndexMetaDataPath(Path path) {
+    private static boolean isIndexMetaDataPath(Path path) {
         return Files.isDirectory(path)
             && path.getFileName().toString().equals(MetaDataStateFormat.STATE_DIR_NAME);
     }