From deed7b51e65e34f956d7f29c37bb46f75e57395b Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 12 Dec 2019 12:31:00 +0100 Subject: [PATCH 1/4] Use only Lucene for metadata storage --- build.gradle | 4 +- .../gateway/GatewayMetaState.java | 139 +----------------- .../gateway/LucenePersistedStateFactory.java | 55 ++++++- .../gateway/MetaStateService.java | 42 +----- .../java/org/elasticsearch/node/Node.java | 25 +++- .../elasticsearch/env/NodeEnvironmentIT.java | 11 +- .../gateway/GatewayIndexStateIT.java | 85 ----------- .../gateway/MetaStateServiceTests.java | 8 +- .../gateway/RecoveryFromGatewayIT.java | 1 + .../PeerRecoveryRetentionLeaseCreationIT.java | 1 + .../RemoveCorruptedShardDataCommandIT.java | 2 + .../indices/recovery/DanglingIndicesIT.java | 2 + .../gateway/MockGatewayMetaState.java | 8 +- 13 files changed, 109 insertions(+), 274 deletions(-) diff --git a/build.gradle b/build.gradle index deb6ee51c7f8d..df1641dcc9efe 100644 --- a/build.gradle +++ b/build.gradle @@ -205,8 +205,8 @@ task verifyVersions { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/issues/48701" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 64fce2e5115d1..10d1613343e37 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -20,30 +20,22 @@ package org.elasticsearch.gateway; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState; import org.elasticsearch.cluster.coordination.InMemoryPersistedState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.index.Index; import org.elasticsearch.plugins.MetaDataUpgrader; import org.elasticsearch.transport.TransportService; @@ -65,7 +57,6 @@ * non-stale state, and master-ineligible nodes receive the real cluster state from the elected master after joining the cluster. */ public class GatewayMetaState implements Closeable { - private static final Logger logger = LogManager.getLogger(GatewayMetaState.class); // Set by calling start() private final SetOnce persistedState = new SetOnce<>(); @@ -81,45 +72,23 @@ public MetaData getMetaData() { } public void start(Settings settings, TransportService transportService, ClusterService clusterService, - MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, + MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader, LucenePersistedStateFactory lucenePersistedStateFactory) { assert persistedState.get() == null : "should only start once, but already have " + persistedState.get(); - if (DiscoveryNode.isMasterNode(settings)) { + if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { try { - persistedState.set(lucenePersistedStateFactory.loadPersistedState((version, metadata) -> + PersistedState ps = lucenePersistedStateFactory.loadPersistedState((version, metadata) -> prepareInitialClusterState(transportService, clusterService, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)) .version(version) .metaData(upgradeMetaDataForMasterEligibleNode(metadata, metaDataIndexUpgradeService, metaDataUpgrader)) - .build()))); + .build())); + persistedState.set(ps); } catch (IOException e) { throw new ElasticsearchException("failed to load metadata", e); } - } - - if (DiscoveryNode.isDataNode(settings)) { - final Tuple manifestClusterStateTuple; - try { - upgradeMetaData(settings, metaStateService, metaDataIndexUpgradeService, metaDataUpgrader); - manifestClusterStateTuple = loadStateAndManifest(ClusterName.CLUSTER_NAME_SETTING.get(settings), metaStateService); - } catch (IOException e) { - throw new ElasticsearchException("failed to load metadata", e); - } - - final IncrementalClusterStateWriter incrementalClusterStateWriter - = new IncrementalClusterStateWriter(settings, clusterService.getClusterSettings(), metaStateService, - manifestClusterStateTuple.v1(), - prepareInitialClusterState(transportService, clusterService, manifestClusterStateTuple.v2()), - transportService.getThreadPool()::relativeTimeInMillis); - - clusterService.addLowPriorityApplier(new GatewayClusterApplier(incrementalClusterStateWriter)); - - if (DiscoveryNode.isMasterNode(settings) == false) { - persistedState.set( - new InMemoryPersistedState(manifestClusterStateTuple.v1().getCurrentTerm(), manifestClusterStateTuple.v2())); - } - } else if (DiscoveryNode.isMasterNode(settings) == false) { + } else { persistedState.set( new InMemoryPersistedState(0L, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)).build())); } @@ -144,70 +113,6 @@ MetaData upgradeMetaDataForMasterEligibleNode(MetaData metaData, return upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); } - // exposed so it can be overridden by tests - void upgradeMetaData(Settings settings, MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) throws IOException { - if (isMasterOrDataNode(settings)) { - try { - final Tuple metaStateAndData = metaStateService.loadFullState(); - final Manifest manifest = metaStateAndData.v1(); - final MetaData metaData = metaStateAndData.v2(); - - // We finished global state validation and successfully checked all indices for backward compatibility - // and found no non-upgradable indices, which means the upgrade can continue. - // Now it's safe to overwrite global and index metadata. - // We don't re-write metadata if it's not upgraded by upgrade plugins, because - // if there is manifest file, it means metadata is properly persisted to all data paths - // if there is no manifest file (upgrade from 6.x to 7.x) metadata might be missing on some data paths, - // but anyway we will re-write it as soon as we receive first ClusterState - final IncrementalClusterStateWriter.AtomicClusterStateWriter writer - = new IncrementalClusterStateWriter.AtomicClusterStateWriter(metaStateService, manifest); - final MetaData upgradedMetaData = upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); - - final long globalStateGeneration; - if (MetaData.isGlobalStateEquals(metaData, upgradedMetaData) == false) { - globalStateGeneration = writer.writeGlobalState("upgrade", upgradedMetaData); - } else { - globalStateGeneration = manifest.getGlobalGeneration(); - } - - Map indices = new HashMap<>(manifest.getIndexGenerations()); - for (IndexMetaData indexMetaData : upgradedMetaData) { - if (metaData.hasIndexMetaData(indexMetaData) == false) { - final long generation = writer.writeIndex("upgrade", indexMetaData); - indices.put(indexMetaData.getIndex(), generation); - } - } - - final Manifest newManifest = new Manifest(manifest.getCurrentTerm(), manifest.getClusterStateVersion(), - globalStateGeneration, indices); - writer.writeManifestAndCleanup("startup", newManifest); - } catch (Exception e) { - logger.error("failed to read or upgrade local state, exiting...", e); - throw e; - } - } - } - - private static Tuple loadStateAndManifest(ClusterName clusterName, - MetaStateService metaStateService) throws IOException { - final long startNS = System.nanoTime(); - final Tuple manifestAndMetaData = metaStateService.loadFullState(); - final Manifest manifest = manifestAndMetaData.v1(); - - final ClusterState clusterState = ClusterState.builder(clusterName) - .version(manifest.getClusterStateVersion()) - .metaData(manifestAndMetaData.v2()).build(); - - logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); - - return Tuple.tuple(manifest, clusterState); - } - - private static boolean isMasterOrDataNode(Settings settings) { - return DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings); - } - /** * Elasticsearch 2.0 removed several deprecated features and as well as support for Lucene 3.x. This method calls * {@link MetaDataIndexUpgradeService} to makes sure that indices are compatible with the current version. The @@ -262,36 +167,4 @@ public void close() throws IOException { IOUtils.close(persistedState.get()); } - private static class GatewayClusterApplier implements ClusterStateApplier { - - private final IncrementalClusterStateWriter incrementalClusterStateWriter; - - private GatewayClusterApplier(IncrementalClusterStateWriter incrementalClusterStateWriter) { - this.incrementalClusterStateWriter = incrementalClusterStateWriter; - } - - @Override - public void applyClusterState(ClusterChangedEvent event) { - if (event.state().blocks().disableStatePersistence()) { - incrementalClusterStateWriter.setIncrementalWrite(false); - return; - } - - try { - // Hack: This is to ensure that non-master-eligible Zen2 nodes always store a current term - // that's higher than the last accepted term. - // TODO: can we get rid of this hack? - if (event.state().term() > incrementalClusterStateWriter.getPreviousManifest().getCurrentTerm()) { - incrementalClusterStateWriter.setCurrentTerm(event.state().term()); - } - - incrementalClusterStateWriter.updateClusterState(event.state()); - incrementalClusterStateWriter.setIncrementalWrite(true); - } catch (WriteStateException e) { - logger.warn("Exception occurred when storing new meta data", e); - } - } - - } - } diff --git a/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java b/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java index f4e7ea8bcad7f..d619d431e0418 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java +++ b/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java @@ -50,8 +50,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.CoordinationState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.logging.Loggers; @@ -125,11 +127,43 @@ public class LucenePersistedStateFactory { private final NodeEnvironment nodeEnvironment; private final NamedXContentRegistry namedXContentRegistry; private final BigArrays bigArrays; + private final LegacyLoader legacyLoader; - public LucenePersistedStateFactory(NodeEnvironment nodeEnvironment, NamedXContentRegistry namedXContentRegistry, BigArrays bigArrays) { + /** + * Allows interacting with legacy metadata + */ + public interface LegacyLoader { + /** + * Loads legacy state + */ + Tuple loadClusterState() throws IOException; + + /** + * Cleans legacy state + */ + void clean() throws IOException; + } + + LucenePersistedStateFactory(NodeEnvironment nodeEnvironment, NamedXContentRegistry namedXContentRegistry, BigArrays bigArrays) { + this(nodeEnvironment, namedXContentRegistry, bigArrays, new LegacyLoader() { + @Override + public Tuple loadClusterState() { + return new Tuple<>(Manifest.empty(), MetaData.EMPTY_META_DATA); + } + + @Override + public void clean() { + + } + }); + } + + public LucenePersistedStateFactory(NodeEnvironment nodeEnvironment, NamedXContentRegistry namedXContentRegistry, BigArrays bigArrays, + LegacyLoader legacyLoader) { this.nodeEnvironment = nodeEnvironment; this.namedXContentRegistry = namedXContentRegistry; this.bigArrays = bigArrays; + this.legacyLoader = legacyLoader; } CoordinationState.PersistedState loadPersistedState(BiFunction clusterStateFromMetaData) @@ -175,7 +209,9 @@ CoordinationState.PersistedState loadPersistedState(BiFunction maxAcceptedTerm + if (bestOnDiskState == NO_ON_DISK_STATE + || acceptedTerm > maxAcceptedTerm || (acceptedTerm == maxAcceptedTerm && (onDiskState.lastAcceptedVersion > bestOnDiskState.lastAcceptedVersion || (onDiskState.lastAcceptedVersion == bestOnDiskState.lastAcceptedVersion) @@ -262,6 +301,14 @@ private OnDiskState loadBestOnDiskState() throws IOException { "] with greater term [" + maxCurrentTermOnDiskState.currentTerm + "]"); } + if (bestOnDiskState == NO_ON_DISK_STATE) { + final Tuple legacyState = legacyLoader.loadClusterState(); + if (legacyState.v1().isEmpty() == false) { + return new OnDiskState(nodeEnvironment.nodeId(), null, legacyState.v1().getCurrentTerm(), + legacyState.v1().getClusterStateVersion(), legacyState.v2()); + } + } + return bestOnDiskState; } diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index 3ce3f8918a190..a2dfc07a2cc01 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -68,7 +68,7 @@ public MetaStateService(NodeEnvironment nodeEnv, NamedXContentRegistry namedXCon * meta state with globalGeneration -1 and empty meta data is returned. * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file. */ - Tuple loadFullState() throws IOException { + public Tuple loadFullState() throws IOException { final Manifest manifest = MANIFEST_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); if (manifest == null) { return loadFullStateBWC(); @@ -184,17 +184,6 @@ List loadIndicesStates(Predicate excludeIndexPathIdsPredi return indexMetaDataList; } - /** - * Loads Manifest file from disk, returns Manifest.empty() if there is no manifest file. - */ - public Manifest loadManifestOrEmpty() throws IOException { - Manifest manifest = MANIFEST_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths()); - if (manifest == null) { - manifest = Manifest.empty(); - } - return manifest; - } - /** * Loads the global state, *without* index state, see {@link #loadFullState()} for that. */ @@ -276,28 +265,13 @@ public void cleanupIndex(Index index, long currentGeneration) { } /** - * Writes index metadata and updates manifest file accordingly. - * Used by tests. + * Removes manifest file, global metadata and all index metadata */ - public void writeIndexAndUpdateManifest(String reason, IndexMetaData metaData) throws IOException { - long generation = writeIndex(reason, metaData); - Manifest manifest = loadManifestOrEmpty(); - Map indices = new HashMap<>(manifest.getIndexGenerations()); - indices.put(metaData.getIndex(), generation); - manifest = new Manifest(manifest.getCurrentTerm(), manifest.getClusterStateVersion(), manifest.getGlobalGeneration(), indices); - writeManifestAndCleanup(reason, manifest); - cleanupIndex(metaData.getIndex(), generation); - } - - /** - * Writes global metadata and updates manifest file accordingly. - * Used by tests. - */ - public void writeGlobalStateAndUpdateManifest(String reason, MetaData metaData) throws IOException { - long generation = writeGlobalState(reason, metaData); - Manifest manifest = loadManifestOrEmpty(); - manifest = new Manifest(manifest.getCurrentTerm(), manifest.getClusterStateVersion(), generation, manifest.getIndexGenerations()); - writeManifestAndCleanup(reason, manifest); - cleanupGlobalState(generation); + public void deleteAll() throws IOException { + MANIFEST_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths()); + META_DATA_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths()); + for (String indexFolderName : nodeEnv.availableIndexFolders()) { + INDEX_META_DATA_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.resolveIndexFolder(indexFolderName)); + } } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 912fbf604109e..d0e8c5efc5572 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.Constants; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Assertions; import org.elasticsearch.Build; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchTimeoutException; @@ -47,6 +48,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; @@ -59,6 +61,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Injector; @@ -94,6 +97,7 @@ import org.elasticsearch.gateway.GatewayModule; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.gateway.LucenePersistedStateFactory; +import org.elasticsearch.gateway.LucenePersistedStateFactory.LegacyLoader; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.index.IndexSettings; @@ -407,7 +411,17 @@ protected Node( .flatMap(Function.identity()).collect(toList())); final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry); final LucenePersistedStateFactory lucenePersistedStateFactory - = new LucenePersistedStateFactory(nodeEnvironment, xContentRegistry, bigArrays); + = new LucenePersistedStateFactory(nodeEnvironment, xContentRegistry, bigArrays, new LegacyLoader() { + @Override + public Tuple loadClusterState() throws IOException { + return metaStateService.loadFullState(); + } + + @Override + public void clean() throws IOException { + metaStateService.deleteAll(); + } + }); // collect engine factory providers from server and from plugins final Collection enginePlugins = pluginsService.filterPlugins(EnginePlugin.class); @@ -691,9 +705,16 @@ public Node start() throws NodeValidationException { // Load (and maybe upgrade) the metadata stored on disk final GatewayMetaState gatewayMetaState = injector.getInstance(GatewayMetaState.class); - gatewayMetaState.start(settings(), transportService, clusterService, injector.getInstance(MetaStateService.class), + gatewayMetaState.start(settings(), transportService, clusterService, injector.getInstance(MetaDataIndexUpgradeService.class), injector.getInstance(MetaDataUpgrader.class), injector.getInstance(LucenePersistedStateFactory.class)); + if (Assertions.ENABLED) { + try { + assert injector.getInstance(MetaStateService.class).loadFullState().v1().isEmpty(); + } catch (IOException e) { + assert false : e; + } + } // we load the global state here (the persistent part of the cluster state stored on disk) to // pass it to the bootstrap checks to allow plugins to enforce certain preconditions based on the recovered state. final MetaData onDiskMetadata = gatewayMetaState.getPersistedState().getLastAcceptedState().metaData(); diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java index 85d2f7e654f1f..16595e3749ae7 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -74,13 +74,10 @@ public Settings onNodeStopped(String nodeName) { .build(); } })); - assertThat(ex.getMessage(), containsString(indexUUID)); assertThat(ex.getMessage(), startsWith("Node is started with " + Node.NODE_DATA_SETTING.getKey() - + "=false and " - + Node.NODE_MASTER_SETTING.getKey() - + "=false, but has index metadata")); + + "=false, but has shard data")); logger.info("--> start the node again with node.data=true and node.master=true"); internalCluster().startNode(dataPathSettings); @@ -191,8 +188,10 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { assertThat(ise.getMessage(), containsString("unexpected folder encountered during data folder upgrade")); Files.delete(badFolder); - final Path conflictingFolder = randomFrom(dataPaths).resolve("indices"); - if (Files.exists(conflictingFolder) == false) { + final Path randomDataPath = randomFrom(dataPaths); + final Path conflictingFolder = randomDataPath.resolve("indices"); + final Path sourceFolder = randomDataPath.resolve("nodes").resolve("0").resolve("indices"); + if (Files.exists(sourceFolder) && Files.exists(conflictingFolder) == false) { Files.createDirectories(conflictingFolder); ise = expectThrows(IllegalStateException.class, () -> internalCluster().startNode(dataPathSettings)); assertThat(ise.getMessage(), containsString("target folder already exists during data folder upgrade")); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index f78a9c874267d..5622ddf47fb81 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -56,7 +56,6 @@ import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -550,90 +549,6 @@ public void testHalfDeletedIndexImport() throws Exception { assertBusy(() -> assertThat(internalCluster().getInstance(NodeEnvironment.class).availableIndexFolders(), empty())); } - public void testOnlyWritesIndexMetaDataFilesOnDataNodes() throws Exception { - final String masterNode = internalCluster().startMasterOnlyNode(); - final String dataNode = internalCluster().startDataOnlyNode(); - final String mixedNode = internalCluster().startNode(); - - createIndex("test", Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 3)) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .build()); - ensureGreen("test"); - - final String indexUUID = client().admin().indices().prepareStats("test").get().getIndex("test").getUuid(); - - final Path[] masterPaths = internalCluster().getInstance(NodeEnvironment.class, masterNode).nodeDataPaths(); - final Path[] dataPaths = internalCluster().getInstance(NodeEnvironment.class, dataNode).nodeDataPaths(); - final Path[] mixedPaths = internalCluster().getInstance(NodeEnvironment.class, mixedNode).nodeDataPaths(); - - for (final Path path : masterPaths) { - assertFalse("master: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER))); - } - for (final Path path : dataPaths) { - assertTrue("data: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - for (final Path path : mixedPaths) { - assertTrue("mixed: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - - logger.info("--> remove shards from data node, to check the index folder is cleaned up"); - assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name", dataNode))); - assertFalse(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus() - .setWaitForNoInitializingShards(true).setWaitForEvents(Priority.LANGUID).get().isTimedOut()); - - for (final Path path : masterPaths) { - assertFalse("master: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER))); - } - for (final Path path : mixedPaths) { - assertTrue("mixed: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - assertBusy(() -> { - for (final Path path : dataPaths) { - assertFalse("data: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - }); - - logger.info("--> remove shards from mixed master/data node, to check the index folder is cleaned up"); - assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() - .put(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name", mixedNode))); - assertFalse(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus() - .setWaitForNoInitializingShards(true).setWaitForEvents(Priority.LANGUID).get().isTimedOut()); - - for (final Path path : masterPaths) { - assertFalse("master: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER))); - } - for (final Path path : dataPaths) { - assertTrue("data: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - assertBusy(() -> { - for (final Path path : mixedPaths) { - assertFalse("mixed: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - }); - - logger.info("--> delete index and check the index folder is cleaned up on all nodes"); - assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder() - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .putNull(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + "._name"))); - ensureGreen("test"); - assertAcked(client().admin().indices().prepareDelete("test")); - - for (final Path path : masterPaths) { - assertFalse("master: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER))); - } - assertBusy(() -> { - for (final Path path : dataPaths) { - assertFalse("data: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - for (final Path path : mixedPaths) { - assertFalse("mixed: " + path, Files.exists(path.resolve(NodeEnvironment.INDICES_FOLDER).resolve(indexUUID))); - } - }); - } - private void restartNodesOnBrokenClusterState(ClusterState.Builder clusterStateBuilder) throws Exception { Map lucenePersistedStateFactories = Stream.of(internalCluster().getNodeNames()) .collect(Collectors.toMap(Function.identity(), diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index 2f34cc4300d2d..069d96d7ddd33 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -155,7 +155,7 @@ public void testLoadFullStateMissingGlobalMetaData() throws IOException { assertThat(loadedMetaData.index("test1"), equalTo(index)); } - public void testLoadFullStateAndUpdate() throws IOException { + public void testLoadFullStateAndUpdateAndClean() throws IOException { IndexMetaData index = indexMetaData("test1"); MetaData metaData = MetaData.builder() .persistentSettings(Settings.builder().put("test1", "value1").build()) @@ -201,5 +201,11 @@ public void testLoadFullStateAndUpdate() throws IOException { assertThat(loadedMetaData.persistentSettings(), equalTo(newMetaData.persistentSettings())); assertThat(loadedMetaData.hasIndex("test1"), equalTo(true)); assertThat(loadedMetaData.index("test1"), equalTo(index)); + + metaStateService.deleteAll(); + manifestAndMetaData = metaStateService.loadFullState(); + assertTrue(manifestAndMetaData.v1().isEmpty()); + metaData = manifestAndMetaData.v2(); + assertTrue(MetaData.isGlobalStateEquals(metaData, MetaData.EMPTY_META_DATA)); } } diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 126c7eeb21ca7..e1ad4969366f2 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -539,6 +539,7 @@ public void assertSyncIdsNotNull() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") public void testStartedShardFoundIfStateNotYetProcessed() throws Exception { // nodes may need to report the shards they processed the initial recovered cluster state from the master final String nodeName = internalCluster().startNode(); diff --git a/server/src/test/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java b/server/src/test/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java index 03842d11b1a40..1d1455348e051 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/PeerRecoveryRetentionLeaseCreationIT.java @@ -42,6 +42,7 @@ protected boolean forbidPrivateIndexSettings() { return false; } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") public void testCanRecoverFromStoreWithoutPeerRecoveryRetentionLease() throws Exception { /* * In a full cluster restart from a version without peer-recovery retention leases, the leases on disk will not include a lease for diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java index b3cee099984c8..32f999a8e91d6 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java @@ -28,6 +28,7 @@ import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.NativeFSLockFactory; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -101,6 +102,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") // TODO remove corrupted shard tool @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java index 31afef73ad450..e642297b96a46 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/DanglingIndicesIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices.recovery; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -30,6 +31,7 @@ import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48701") // TODO add dangling indices support @ClusterScope(numDataNodes = 0, scope = ESIntegTestCase.Scope.TEST) public class DanglingIndicesIT extends ESIntegTestCase { private static final String INDEX_NAME = "test-idx-1"; diff --git a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java index 8e2e5f0da4ca4..6e51051862e8d 100644 --- a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java +++ b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java @@ -49,12 +49,6 @@ public MockGatewayMetaState(DiscoveryNode localNode) { this.localNode = localNode; } - @Override - void upgradeMetaData(Settings settings, MetaStateService metaStateService, MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) { - // MetaData upgrade is tested in GatewayMetaStateTests, we override this method to NOP to make mocking easier - } - @Override MetaData upgradeMetaDataForMasterEligibleNode(MetaData metaData, MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) { @@ -74,7 +68,7 @@ public void start(Settings settings, NodeEnvironment nodeEnvironment, NamedXCont final ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()) .thenReturn(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); - start(settings, transportService, clusterService, new MetaStateService(nodeEnvironment, xContentRegistry), + start(settings, transportService, clusterService, null, null, new LucenePersistedStateFactory(nodeEnvironment, xContentRegistry, BigArrays.NON_RECYCLING_INSTANCE)); } } From cc64a19711753d19af9b70c2afbf2c13b714fd25 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Thu, 12 Dec 2019 21:09:54 +0100 Subject: [PATCH 2/4] review comments --- .../gateway/GatewayMetaState.java | 14 ++++++------- .../gateway/LucenePersistedStateFactory.java | 6 +++--- .../gateway/MetaStateService.java | 3 ++- .../gateway/MetaDataWriteDataNodesIT.java | 21 +++++++++++++++++-- .../gateway/MockGatewayMetaState.java | 4 ++-- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 10d1613343e37..cfffc89fac3b1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -78,13 +78,13 @@ public void start(Settings settings, TransportService transportService, ClusterS if (DiscoveryNode.isMasterNode(settings) || DiscoveryNode.isDataNode(settings)) { try { - PersistedState ps = lucenePersistedStateFactory.loadPersistedState((version, metadata) -> + persistedState.set(lucenePersistedStateFactory.loadPersistedState((version, metadata) -> prepareInitialClusterState(transportService, clusterService, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings)) .version(version) - .metaData(upgradeMetaDataForMasterEligibleNode(metadata, metaDataIndexUpgradeService, metaDataUpgrader)) - .build())); - persistedState.set(ps); + .metaData(upgradeMetaDataForNode(metadata, metaDataIndexUpgradeService, metaDataUpgrader)) + .build())) + ); } catch (IOException e) { throw new ElasticsearchException("failed to load metadata", e); } @@ -107,9 +107,9 @@ ClusterState prepareInitialClusterState(TransportService transportService, Clust } // exposed so it can be overridden by tests - MetaData upgradeMetaDataForMasterEligibleNode(MetaData metaData, - MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) { + MetaData upgradeMetaDataForNode(MetaData metaData, + MetaDataIndexUpgradeService metaDataIndexUpgradeService, + MetaDataUpgrader metaDataUpgrader) { return upgradeMetaData(metaData, metaDataIndexUpgradeService, metaDataUpgrader); } diff --git a/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java b/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java index d619d431e0418..2e1e22ebf445a 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java +++ b/server/src/main/java/org/elasticsearch/gateway/LucenePersistedStateFactory.java @@ -206,12 +206,11 @@ CoordinationState.PersistedState loadPersistedState(BiFunction legacyState = legacyLoader.loadClusterState(); if (legacyState.v1().isEmpty() == false) { return new OnDiskState(nodeEnvironment.nodeId(), null, legacyState.v1().getCurrentTerm(), diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index a2dfc07a2cc01..b8f2d1b945cb6 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -271,7 +271,8 @@ public void deleteAll() throws IOException { MANIFEST_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths()); META_DATA_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.nodeDataPaths()); for (String indexFolderName : nodeEnv.availableIndexFolders()) { - INDEX_META_DATA_FORMAT.cleanupOldFiles(Long.MAX_VALUE, nodeEnv.resolveIndexFolder(indexFolderName)); + // delete meta state directories of indices + MetaDataStateFormat.deleteMetaState(nodeEnv.resolveIndexFolder(indexFolderName)); } } } diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java index d0d42b6fef556..83ba4cc638266 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java @@ -56,8 +56,8 @@ public void testMetaWrittenAlsoOnDataNode() throws Exception { assertIndexInMetaState(masterNode, "test"); } - public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { - // this test checks that the index state is removed from a data only node once all shards have been allocated away from it + public void testIndexFilesAreRemovedIfAllShardsFromIndexRemoved() throws Exception { + // this test checks that the index data is removed from a data only node once all shards have been allocated away from it String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); List nodeNames= internalCluster().startDataOnlyNodes(2); String node1 = nodeNames.get(0); @@ -70,8 +70,10 @@ public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { ensureGreen(); assertIndexInMetaState(node1, index); Index resolveIndex = resolveIndex(index); + assertIndexDirectoryExists(node1, resolveIndex); assertIndexDirectoryDeleted(node2, resolveIndex); assertIndexInMetaState(masterNode, index); + assertIndexDirectoryDeleted(masterNode, resolveIndex); logger.debug("relocating index..."); client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder() @@ -80,7 +82,13 @@ public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { ensureGreen(); assertIndexDirectoryDeleted(node1, resolveIndex); assertIndexInMetaState(node2, index); + assertIndexDirectoryExists(node2, resolveIndex); assertIndexInMetaState(masterNode, index); + assertIndexDirectoryDeleted(masterNode, resolveIndex); + + client().admin().indices().prepareDelete(index).get(); + assertIndexDirectoryDeleted(node1, resolveIndex); + assertIndexDirectoryDeleted(node2, resolveIndex); } @SuppressWarnings("unchecked") @@ -165,6 +173,15 @@ protected void assertIndexDirectoryDeleted(final String nodeName, final Index in ); } + protected void assertIndexDirectoryExists(final String nodeName, final Index index) throws Exception { + assertBusy(() -> { + logger.info("checking if index directory exists..."); + assertTrue("Expecting index directory of " + index + " to exist on node " + nodeName, + indexDirectoryExists(nodeName, index)); + } + ); + } + protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception { assertBusy(() -> { logger.info("checking if meta state exists..."); diff --git a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java index 6e51051862e8d..79bfd5512c2c2 100644 --- a/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java +++ b/test/framework/src/main/java/org/elasticsearch/gateway/MockGatewayMetaState.java @@ -50,8 +50,8 @@ public MockGatewayMetaState(DiscoveryNode localNode) { } @Override - MetaData upgradeMetaDataForMasterEligibleNode(MetaData metaData, MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) { + MetaData upgradeMetaDataForNode(MetaData metaData, MetaDataIndexUpgradeService metaDataIndexUpgradeService, + MetaDataUpgrader metaDataUpgrader) { // MetaData upgrade is tested in GatewayMetaStateTests, we override this method to NOP to make mocking easier return metaData; } From 5a0558a8e3f67049c5b49ab6099184fb50eee209 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 13 Dec 2019 08:56:21 +0100 Subject: [PATCH 3/4] disable packaging test --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 9691636f5ccda..041b1791ee4f4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.packaging.util.ServerUtils; import org.elasticsearch.packaging.util.Shell.Result; import org.junit.BeforeClass; +import org.junit.Ignore; import java.nio.file.Files; import java.nio.file.Path; @@ -382,6 +383,7 @@ public void test92ElasticsearchNodeCliPackaging() throws Exception { } } + @Ignore("https://github.com/elastic/elasticsearch/issues/48701") // TODO unsafe bootstrapping public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Exception { Path relativeDataPath = installation.data.relativize(installation.home); append(installation.config("elasticsearch.yml"), "path.data: " + relativeDataPath); From 2dee55ce36873d0d20370f2f2e5ef718dc612707 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 13 Dec 2019 11:26:05 +0100 Subject: [PATCH 4/4] remove unneeded info logging --- .../gateway/MetaDataWriteDataNodesIT.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java index 83ba4cc638266..6cadf896453c3 100644 --- a/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java @@ -165,26 +165,19 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { } protected void assertIndexDirectoryDeleted(final String nodeName, final Index index) throws Exception { - assertBusy(() -> { - logger.info("checking if index directory exists..."); - assertFalse("Expecting index directory of " + index + " to be deleted from node " + nodeName, - indexDirectoryExists(nodeName, index)); - } + assertBusy(() -> assertFalse("Expecting index directory of " + index + " to be deleted from node " + nodeName, + indexDirectoryExists(nodeName, index)) ); } protected void assertIndexDirectoryExists(final String nodeName, final Index index) throws Exception { - assertBusy(() -> { - logger.info("checking if index directory exists..."); - assertTrue("Expecting index directory of " + index + " to exist on node " + nodeName, - indexDirectoryExists(nodeName, index)); - } + assertBusy(() -> assertTrue("Expecting index directory of " + index + " to exist on node " + nodeName, + indexDirectoryExists(nodeName, index)) ); } protected void assertIndexInMetaState(final String nodeName, final String indexName) throws Exception { assertBusy(() -> { - logger.info("checking if meta state exists..."); try { assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName));