Skip to content

Commit 678aeb7

Browse files
ywelschywelsch
authored
Make elasticsearch-node tools custom metadata-aware (#48390)
The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently unaware of plugins and will therefore drop custom metadata from the cluster state once the state is written out again (as it skips over the custom metadata that it can't read). This commit preserves unknown customs when editing on-disk metadata through the elasticsearch-node command-line tools.
1 parent f400b18 commit 678aeb7

File tree

14 files changed

+153
-54
lines changed

14 files changed

+153
-54
lines changed

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -424,23 +424,23 @@ public synchronized void start() {
424424

425425
if (plugins.isEmpty() == false) {
426426
logToProcessStdout("Installing " + plugins.size() + " plugins");
427-
plugins.forEach(plugin -> runElaticsearchBinScript(
427+
plugins.forEach(plugin -> runElasticsearchBinScript(
428428
"elasticsearch-plugin",
429429
"install", "--batch", plugin.toString())
430430
);
431431
}
432432

433433
if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
434434
LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
435-
runElaticsearchBinScript(
435+
runElasticsearchBinScript(
436436
"elasticsearch-plugin",
437437
"install", "--batch", "x-pack"
438438
);
439439
}
440440

441441
if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
442442
logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
443-
runElaticsearchBinScript("elasticsearch-keystore", "create");
443+
runElasticsearchBinScript("elasticsearch-keystore", "create");
444444

445445
keystoreSettings.forEach((key, value) ->
446446
runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
@@ -452,7 +452,7 @@ public synchronized void start() {
452452
if (file.exists() == false) {
453453
throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
454454
}
455-
runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
455+
runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
456456
}
457457
}
458458

@@ -467,7 +467,7 @@ public synchronized void start() {
467467
if (credentials.isEmpty() == false) {
468468
logToProcessStdout("Setting up " + credentials.size() + " users");
469469

470-
credentials.forEach(paramMap -> runElaticsearchBinScript(
470+
credentials.forEach(paramMap -> runElasticsearchBinScript(
471471
getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
472472
paramMap.entrySet().stream()
473473
.flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -663,7 +663,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
663663
}
664664
}
665665

666-
private void runElaticsearchBinScript(String tool, String... args) {
666+
private void runElasticsearchBinScript(String tool, String... args) {
667667
runElasticsearchBinScriptWithInput("", tool, args);
668668
}
669669

‎libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -819,7 +819,7 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I
819819
} else if (value instanceof Map) {
820820
@SuppressWarnings("unchecked")
821821
final Map<String, ?> valueMap = (Map<String, ?>) value;
822-
map(valueMap, ensureNoSelfReferences);
822+
map(valueMap, ensureNoSelfReferences, true);
823823
} else if (value instanceof Iterable) {
824824
value((Iterable<?>) value, ensureNoSelfReferences);
825825
} else if (value instanceof Object[]) {
@@ -867,10 +867,15 @@ public XContentBuilder field(String name, Map<String, Object> values) throws IOE
867867
}
868868

869869
public XContentBuilder map(Map<String, ?> values) throws IOException {
870-
return map(values, true);
870+
return map(values, true, true);
871871
}
872872

873-
private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences) throws IOException {
873+
/** writes a map without the start object and end object headers */
874+
public XContentBuilder mapContents(Map<String, ?> values) throws IOException {
875+
return map(values, true, false);
876+
}
877+
878+
private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException {
874879
if (values == null) {
875880
return nullValue();
876881
}
@@ -881,13 +886,17 @@ private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReference
881886
ensureNoSelfReferences(values);
882887
}
883888

884-
startObject();
889+
if (writeStartAndEndHeaders) {
890+
startObject();
891+
}
885892
for (Map.Entry<String, ?> value : values.entrySet()) {
886893
field(value.getKey());
887894
// pass ensureNoSelfReferences=false as we already performed the check at a higher level
888895
unknownValue(value.getValue(), false);
889896
}
890-
endObject();
897+
if (writeStartAndEndHeaders) {
898+
endObject();
899+
}
891900
return this;
892901
}
893902

‎server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,19 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception {
8888

8989
/** Create an {@link Environment} for the command to use. Overrideable for tests. */
9090
protected Environment createEnv(final Map<String, String> settings) throws UserException {
91+
return createEnv(Settings.EMPTY, settings);
92+
}
93+
94+
/** Create an {@link Environment} for the command to use. Overrideable for tests. */
95+
protected final Environment createEnv(final Settings baseSettings, final Map<String, String> settings) throws UserException {
9196
final String esPathConf = System.getProperty("es.path.conf");
9297
if (esPathConf == null) {
9398
throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set");
9499
}
95-
return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings,
96-
getConfigPath(esPathConf),
97-
// HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
98-
() -> System.getenv("HOSTNAME"));
100+
return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings,
101+
getConfigPath(esPathConf),
102+
// HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
103+
() -> System.getenv("HOSTNAME"));
99104
}
100105

101106
@SuppressForbidden(reason = "need path to construct environment")

‎server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.elasticsearch.ElasticsearchException;
2727
import org.elasticsearch.cli.EnvironmentAwareCommand;
2828
import org.elasticsearch.cli.Terminal;
29-
import org.elasticsearch.cluster.ClusterModule;
3029
import org.elasticsearch.cluster.metadata.Manifest;
3130
import org.elasticsearch.cluster.metadata.MetaData;
3231
import org.elasticsearch.common.collect.Tuple;
@@ -42,7 +41,6 @@
4241

4342
public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
4443
private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
45-
protected final NamedXContentRegistry namedXContentRegistry;
4644
protected static final String DELIMITER = "------------------------------------------------------------------------\n";
4745

4846
static final String STOP_WARNING_MSG =
@@ -61,7 +59,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
6159

6260
public ElasticsearchNodeCommand(String description) {
6361
super(description);
64-
namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
6562
}
6663

6764
protected void processNodePaths(Terminal terminal, OptionSet options, Environment env) throws IOException {
@@ -80,7 +77,7 @@ protected void processNodePaths(Terminal terminal, OptionSet options, Environmen
8077

8178
protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException {
8279
terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
83-
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
80+
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
8481

8582
if (manifest == null) {
8683
throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
@@ -89,8 +86,8 @@ protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataP
8986
throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
9087
}
9188
terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
92-
final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
93-
dataPaths);
89+
final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration(
90+
logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths);
9491
if (metaData == null) {
9592
throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
9693
}

‎server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.common.collect.Tuple;
2929
import org.elasticsearch.common.settings.Setting;
3030
import org.elasticsearch.common.settings.Settings;
31+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3132
import org.elasticsearch.env.Environment;
3233
import org.elasticsearch.env.NodeMetaData;
3334
import org.elasticsearch.node.Node;
@@ -84,7 +85,7 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
8485

8586
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
8687
terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
87-
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
88+
final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
8889
if (nodeMetaData == null) {
8990
throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
9091
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import org.elasticsearch.common.settings.Setting;
4646
import org.elasticsearch.common.settings.Setting.Property;
4747
import org.elasticsearch.common.settings.Settings;
48-
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4948
import org.elasticsearch.common.xcontent.ToXContent;
5049
import org.elasticsearch.common.xcontent.ToXContentFragment;
5150
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1421,8 +1420,6 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx
14211420

14221421
@Override
14231422
public IndexMetaData fromXContent(XContentParser parser) throws IOException {
1424-
assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
1425-
: "loading index metadata requires a working named xcontent registry";
14261423
return Builder.fromXContent(parser);
14271424
}
14281425
};

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java‎

Lines changed: 67 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -755,7 +755,7 @@ public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
755755
}
756756

757757
public static MetaData fromXContent(XContentParser parser) throws IOException {
758-
return Builder.fromXContent(parser);
758+
return Builder.fromXContent(parser, false);
759759
}
760760

761761
@Override
@@ -1277,7 +1277,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
12771277
builder.endObject();
12781278
}
12791279

1280-
public static MetaData fromXContent(XContentParser parser) throws IOException {
1280+
public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException {
12811281
Builder builder = new Builder();
12821282

12831283
// we might get here after the meta-data element, or on a fresh parser
@@ -1327,8 +1327,13 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
13271327
Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
13281328
builder.putCustom(custom.getWriteableName(), custom);
13291329
} catch (NamedObjectNotFoundException ex) {
1330-
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
1331-
parser.skipChildren();
1330+
if (preserveUnknownCustoms) {
1331+
logger.warn("Adding unknown custom object with type {}", currentFieldName);
1332+
builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered()));
1333+
} else {
1334+
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
1335+
parser.skipChildren();
1336+
}
13321337
}
13331338
}
13341339
} else if (token.isValue()) {
@@ -1349,6 +1354,45 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
13491354
}
13501355
}
13511356

1357+
public static class UnknownGatewayOnlyCustom implements Custom {
1358+
1359+
private final Map<String, Object> contents;
1360+
1361+
UnknownGatewayOnlyCustom(Map<String, Object> contents) {
1362+
this.contents = contents;
1363+
}
1364+
1365+
@Override
1366+
public EnumSet<XContentContext> context() {
1367+
return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
1368+
}
1369+
1370+
@Override
1371+
public Diff<Custom> diff(Custom previousState) {
1372+
throw new UnsupportedOperationException();
1373+
}
1374+
1375+
@Override
1376+
public String getWriteableName() {
1377+
throw new UnsupportedOperationException();
1378+
}
1379+
1380+
@Override
1381+
public Version getMinimalSupportedVersion() {
1382+
throw new UnsupportedOperationException();
1383+
}
1384+
1385+
@Override
1386+
public void writeTo(StreamOutput out) throws IOException {
1387+
throw new UnsupportedOperationException();
1388+
}
1389+
1390+
@Override
1391+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
1392+
return builder.mapContents(contents);
1393+
}
1394+
}
1395+
13521396
private static final ToXContent.Params FORMAT_PARAMS;
13531397
static {
13541398
Map<String, String> params = new HashMap<>(2);
@@ -1360,16 +1404,25 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
13601404
/**
13611405
* State format for {@link MetaData} to write to and load from disk
13621406
*/
1363-
public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
1407+
public static final MetaDataStateFormat<MetaData> FORMAT = createMetaDataStateFormat(false);
13641408

1365-
@Override
1366-
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
1367-
Builder.toXContent(state, builder, FORMAT_PARAMS);
1368-
}
1409+
/**
1410+
* Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs
1411+
*/
1412+
public static final MetaDataStateFormat<MetaData> FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true);
13691413

1370-
@Override
1371-
public MetaData fromXContent(XContentParser parser) throws IOException {
1372-
return Builder.fromXContent(parser);
1373-
}
1374-
};
1414+
private static MetaDataStateFormat<MetaData> createMetaDataStateFormat(boolean preserveUnknownCustoms) {
1415+
return new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
1416+
1417+
@Override
1418+
public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
1419+
Builder.toXContent(state, builder, FORMAT_PARAMS);
1420+
}
1421+
1422+
@Override
1423+
public MetaData fromXContent(XContentParser parser) throws IOException {
1424+
return Builder.fromXContent(parser, preserveUnknownCustoms);
1425+
}
1426+
};
1427+
}
13751428
}

‎server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.cluster.metadata.Manifest;
3030
import org.elasticsearch.cluster.node.DiscoveryNode;
3131
import org.elasticsearch.common.settings.Settings;
32+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3233
import org.elasticsearch.core.internal.io.IOUtils;
3334
import org.elasticsearch.gateway.WriteStateException;
3435

@@ -165,7 +166,7 @@ private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) {
165166
indexPaths[i] = nodePaths[i].resolve(uuid);
166167
}
167168
try {
168-
IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths);
169+
IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths);
169170
return metaData.getIndex().getName();
170171
} catch (Exception e) {
171172
return "no name for uuid: " + uuid + ": " + e;
@@ -194,7 +195,7 @@ private void rewriteManifest(Terminal terminal, Manifest manifest, Path[] dataPa
194195

195196
private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException {
196197
terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest");
197-
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
198+
final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
198199

199200
if (manifest == null) {
200201
terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE);

‎server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.Version;
2626
import org.elasticsearch.cli.Terminal;
2727
import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand;
28+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2829

2930
import java.io.IOException;
3031
import java.nio.file.Path;
@@ -74,7 +75,7 @@ public OverrideNodeVersionCommand() {
7475
protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
7576
final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new);
7677
final NodeMetaData nodeMetaData
77-
= new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths);
78+
= new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths);
7879
if (nodeMetaData == null) {
7980
throw new ElasticsearchException(NO_METADATA_MESSAGE);
8081
}

‎server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ public void testUnknownFieldClusterMetaData() throws IOException {
365365
.endObject()
366366
.endObject());
367367
try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) {
368-
MetaData.Builder.fromXContent(parser);
368+
MetaData.Builder.fromXContent(parser, randomBoolean());
369369
fail();
370370
} catch (IllegalArgumentException e) {
371371
assertEquals("Unexpected field [random]", e.getMessage());

0 commit comments

Comments
 (0)