From 0ce5addb976c0ae4bca2fb9d7d0c3a9e7c44afa1 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 3 Aug 2019 07:44:40 -0400 Subject: [PATCH 1/5] Introduce formal node ML role This commit builds on the ability for plugins to introduce new roles to add a formal node ML role. --- docs/reference/cat/nodes.asciidoc | 2 +- docs/reference/getting-started.asciidoc | 4 ++-- .../elasticsearch/xpack/ml/MachineLearning.java | 17 +++++++++++++++++ .../xpack/ml/job/JobNodeSelectorTests.java | 4 ---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 387044e6a95e0..795a893d620e0 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -18,7 +18,7 @@ ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master -------------------------------------------------- // TESTRESPONSE[s/3.07/(\\d+\\.\\d+( \\d+\\.\\d+ (\\d+\\.\\d+)?)?)?/] // TESTRESPONSE[s/65 99 42/\\d+ \\d+ \\d+/] -// TESTRESPONSE[s/[*]/[*]/ s/mJw06l1/.+/ non_json] +// TESTRESPONSE[s/dim/.+/ s/[*]/[*]/ s/mJw06l1/.+/ non_json] The first few columns (`ip`, `heap.percent`, `ram.percent`, `cpu`, `load_*`) tell you where your nodes live and give a quick picture of performance stats. diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index bb4fc68d56a95..bb7a8633295b2 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -247,10 +247,10 @@ And the response: [source,txt] -------------------------------------------------- ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name -127.0.0.1 10 5 5 4.46 dim * PB2SGZY +127.0.0.1 10 5 5 4.46 dim * PB2SGZY -------------------------------------------------- // TESTRESPONSE[s/10 5 5 4.46/\\d+ \\d+ \\d+ (\\d+\\.\\d+)? (\\d+\\.\\d+)? (\\d+\.\\d+)?/] -// TESTRESPONSE[s/[*]/[*]/ s/PB2SGZY/.+/ non_json] +// TESTRESPONSE[s/dim/.+/ s/[*]/[*]/ s/PB2SGZY/.+/ non_json] Here, we can see our one node named "PB2SGZY", which is the single node that is currently in our cluster. diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 786e0385a8189..2bdbe7848a2e5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -19,6 +19,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.service.ClusterService; @@ -280,6 +281,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -300,6 +302,21 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu public static final Setting ML_ENABLED = Setting.boolSetting("node.ml", XPackSettings.MACHINE_LEARNING_ENABLED, Property.NodeScope); + + static DiscoveryNodeRole ML_ROLE = new DiscoveryNodeRole("ml", "l") { + + @Override + protected Setting roleSetting() { + return ML_ENABLED; + } + + }; + + @Override + public Set getRoles() { + return Collections.singleton(ML_ROLE); + } + // This is not used in v7 and higher, but users are still prevented from setting it directly to avoid confusion private static final String PRE_V7_ML_ENABLED_NODE_ATTR = "ml.enabled"; public static final String MAX_OPEN_JOBS_NODE_ATTR = "ml.max_open_jobs"; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java index f0388abd5d5fd..896a690be8da7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobNodeSelectorTests.java @@ -84,10 +84,6 @@ public void testNodeNameAndMlAttributes() { node = new DiscoveryNode(null, "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); assertEquals("{_node_id1}{ml.machine_memory=5}", JobNodeSelector.nodeNameAndMlAttributes(node)); - - attributes.put("node.ml", "true"); - node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); - assertEquals("{_node_name1}{ml.machine_memory=5}{node.ml=true}", JobNodeSelector.nodeNameAndMlAttributes(node)); } public void testSelectLeastLoadedMlNode_byCount() { From 564a5d2687fed532d3e7f31aed7cf15d3fff8c40 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 3 Aug 2019 15:44:21 -0400 Subject: [PATCH 2/5] Fix Docker test --- .../src/test/resources/rest-api-spec/test/11_nodes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml index 1004e19b6b5ae..9c46c1c0ccd77 100644 --- a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml +++ b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml @@ -6,8 +6,8 @@ - match: $body: | - / #ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name - ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + / #ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name + ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: @@ -15,8 +15,8 @@ - match: $body: | - /^ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load_1m \s+ load_5m \s+ load_15m \s+ node\.role \s+ master \s+ name \n - ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + /^ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load_1m \s+ load_5m \s+ load_15m \s+ node\.role \s+ master \s+ name \n + ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: From 49e658f05bd0e3f1bda356984ef643451adbaff8 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 3 Aug 2019 20:45:38 -0400 Subject: [PATCH 3/5] Fix cat nodes test --- .../resources/rest-api-spec/test/cat.nodes/10_basic.yml | 8 ++++---- .../java/org/elasticsearch/index/shard/StoreRecovery.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml index 7663b693105c1..bde9e1d26826f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yml @@ -6,8 +6,8 @@ - match: $body: | - / #ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name - ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + / #ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name + ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: @@ -15,8 +15,8 @@ - match: $body: | - /^ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load_1m \s+ load_5m \s+ load_15m \s+ node\.role \s+ master \s+ name \n - ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dmi]{1,3}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + /^ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load_1m \s+ load_5m \s+ load_15m \s+ node\.role \s+ master \s+ name \n + ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[dilm]{1,4}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ - do: cat.nodes: diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index da4375f038443..3a745bfffb6ea 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -166,7 +166,7 @@ void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory ta } try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) { - writer.addIndexes(sources); + writer.addIndexes(sources);/ if (split) { writer.deleteDocuments(new ShardSplittingQuery(indexMetaData, shardId, hasNested)); } From f2058d3f3cf5bc0b369541fe660d907c3c018f01 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 3 Aug 2019 20:49:36 -0400 Subject: [PATCH 4/5] Remove accidental insert --- .../main/java/org/elasticsearch/index/shard/StoreRecovery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 3a745bfffb6ea..da4375f038443 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -166,7 +166,7 @@ void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory ta } try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) { - writer.addIndexes(sources);/ + writer.addIndexes(sources); if (split) { writer.deleteDocuments(new ShardSplittingQuery(indexMetaData, shardId, hasNested)); } From c7122e5adffc7f8450c83f834ab8bb232804c197 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 6 Aug 2019 06:50:53 -0400 Subject: [PATCH 5/5] Feedback --- .../main/java/org/elasticsearch/xpack/ml/MachineLearning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 2bdbe7848a2e5..67ce32f128e94 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -303,7 +303,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu public static final Setting ML_ENABLED = Setting.boolSetting("node.ml", XPackSettings.MACHINE_LEARNING_ENABLED, Property.NodeScope); - static DiscoveryNodeRole ML_ROLE = new DiscoveryNodeRole("ml", "l") { + public static final DiscoveryNodeRole ML_ROLE = new DiscoveryNodeRole("ml", "l") { @Override protected Setting roleSetting() {