From 8127a06b2ec40fd5f31a6915143386dd853abc2b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 18 Mar 2016 13:09:20 +0100 Subject: [PATCH] Recover broken IndexMetaData as closed Today if something is wrong with the IndexMetaData we detect it very late and most of the time if that happens we already allocated the index and get endless loops and full log files on data-nodes. This change tries to verify IndexService creattion during initial state recovery on the master and if the recovery fails the index is imported as `closed` and won't be allocated at all. Closes #17187 --- .../resources/checkstyle_suppressions.xml | 1 - .../metadata/MetaDataIndexStateService.java | 17 ++- .../org/elasticsearch/gateway/Gateway.java | 29 +++- .../gateway/GatewayMetaState.java | 6 +- .../elasticsearch/gateway/GatewayService.java | 8 +- .../elasticsearch/indices/IndicesService.java | 79 ++++++++--- .../gateway/GatewayIndexStateIT.java | 128 ++++++++++++++++++ .../gateway/GatewayServiceTests.java | 3 +- 8 files changed, 238 insertions(+), 33 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 46ddcb0ad0e07..72e10cb93980c 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -419,7 +419,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java index 71962d6356a3a..e6e7084e4d96e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexClusterStateUpdateRequest; @@ -37,6 +38,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.snapshots.SnapshotsService; @@ -59,10 +62,16 @@ public class MetaDataIndexStateService extends AbstractComponent { private final AllocationService allocationService; private final MetaDataIndexUpgradeService metaDataIndexUpgradeService; + private final NodeServicesProvider nodeServiceProvider; + private final IndicesService indicesService; @Inject - public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService, MetaDataIndexUpgradeService metaDataIndexUpgradeService) { + public MetaDataIndexStateService(Settings settings, ClusterService clusterService, AllocationService allocationService, + MetaDataIndexUpgradeService metaDataIndexUpgradeService, + NodeServicesProvider nodeServicesProvider, IndicesService indicesService) { super(settings); + this.nodeServiceProvider = nodeServicesProvider; + this.indicesService = indicesService; this.clusterService = clusterService; this.allocationService = allocationService; this.metaDataIndexUpgradeService = metaDataIndexUpgradeService; @@ -162,6 +171,12 @@ public ClusterState execute(ClusterState currentState) { // The index might be closed because we couldn't import it due to old incompatible version // We need to check that this index can be upgraded to the current version indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData); + try { + indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData); + } catch (Exception e) { + throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e); + } + mdBuilder.put(indexMetaData, true); blocksBuilder.removeIndexBlock(indexName, INDEX_CLOSED_BLOCK); } diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java index f5d38112c4fd7..54c270e207215 100644 --- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -35,8 +35,14 @@ import org.elasticsearch.discovery.Discovery; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.indices.IndicesService; +import java.io.IOException; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; import java.util.function.Supplier; /** @@ -53,10 +59,15 @@ public class Gateway extends AbstractComponent implements ClusterStateListener { private final TransportNodesListGatewayMetaState listGatewayMetaState; private final Supplier minimumMasterNodesProvider; + private final IndicesService indicesService; + private final NodeServicesProvider nodeServicesProvider; public Gateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, GatewayMetaState metaState, - TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) { + TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery, + NodeServicesProvider nodeServicesProvider, IndicesService indicesService) { super(settings); + this.nodeServicesProvider = nodeServicesProvider; + this.indicesService = indicesService; this.clusterService = clusterService; this.nodeEnv = nodeEnv; this.metaState = metaState; @@ -66,9 +77,9 @@ public Gateway(Settings settings, ClusterService clusterService, NodeEnvironment } public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException { - ObjectHashSet nodesIds = new ObjectHashSet<>(clusterService.state().nodes().masterNodes().keys()); - logger.trace("performing state recovery from {}", nodesIds); - TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds.toArray(String.class), null).actionGet(); + String[] nodesIds = clusterService.state().nodes().masterNodes().keys().toArray(String.class); + logger.trace("performing state recovery from {}", Arrays.toString(nodesIds)); + TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet(); int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get()); @@ -129,7 +140,17 @@ public void performStateRecovery(final GatewayStateRecoveredListener listener) t if (electedIndexMetaData != null) { if (indexMetaDataCount < requiredAllocation) { logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation); + } // TODO if this logging statement is correct then we are missing an else here + try { + if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) { + // verify that we can actually create this index - if not we recover it as closed with lots of warn logs + indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData); + } + } catch (Exception e) { + logger.warn("recovering index {} failed - recovering as closed", e, electedIndexMetaData.getIndex()); + electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build(); } + metaDataBuilder.put(electedIndexMetaData, false); } } diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 950b4351e1d49..1da82468997ae 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -86,8 +86,8 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer if (DiscoveryNode.masterNode(settings) || DiscoveryNode.dataNode(settings)) { try { ensureNoPre019State(); - pre20Upgrade(); IndexFolderUpgrader.upgradeIndicesIfNeeded(settings, nodeEnv); + upgradeMetaData(); long startNS = System.nanoTime(); metaStateService.loadFullState(); logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); @@ -222,7 +222,7 @@ private void ensureNoPre019State() throws Exception { * MetaDataIndexUpgradeService might also update obsolete settings if needed. When this happens we rewrite * index metadata with new settings. */ - private void pre20Upgrade() throws Exception { + private void upgradeMetaData() throws Exception { MetaData metaData = loadMetaState(); List updateIndexMetaData = new ArrayList<>(); for (IndexMetaData indexMetaData : metaData) { @@ -235,7 +235,7 @@ private void pre20Upgrade() throws Exception { // means the upgrade can continue. Now it's safe to overwrite index metadata with the new version. for (IndexMetaData indexMetaData : updateIndexMetaData) { // since we still haven't upgraded the index folders, we write index state in the old folder - metaStateService.writeIndex("upgrade", indexMetaData, nodeEnv.resolveIndexFolder(indexMetaData.getIndex().getName())); + metaStateService.writeIndex("upgrade", indexMetaData, nodeEnv.resolveIndexFolder(indexMetaData.getIndex().getUUID())); } } diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java index 16d67a84c4a30..7dcc45f1c0a9c 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -43,6 +43,8 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.discovery.Discovery; import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -95,9 +97,11 @@ public class GatewayService extends AbstractLifecycleComponent i @Inject public GatewayService(Settings settings, AllocationService allocationService, ClusterService clusterService, ThreadPool threadPool, NodeEnvironment nodeEnvironment, GatewayMetaState metaState, - TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery) { + TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery, + NodeServicesProvider nodeServicesProvider, IndicesService indicesService) { super(settings); - this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery); + this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery, + nodeServicesProvider, indicesService); this.allocationService = allocationService; this.clusterService = clusterService; this.threadPool = threadPool; diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index b43d33b1bd962..a0dba7089a91b 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices; +import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.CollectionUtil; @@ -33,6 +34,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -66,6 +68,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; @@ -74,6 +77,7 @@ import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; +import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.IndexStoreConfig; @@ -88,9 +92,11 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.threadpool.ThreadPool; +import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; @@ -324,6 +330,7 @@ public IndexService indexServiceSafe(Index index) { * @throws IndexAlreadyExistsException if the index already exists. */ public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List builtInListeners) throws IOException { + if (!lifecycle.started()) { throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed"); } @@ -331,37 +338,22 @@ public synchronized IndexService createIndex(final NodeServicesProvider nodeServ throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]"); } final Index index = indexMetaData.getIndex(); - final Predicate indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state()); - final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting); if (hasIndex(index)) { throw new IndexAlreadyExistsException(index); } - logger.debug("creating Index [{}], shards [{}]/[{}{}]", - indexMetaData.getIndex(), - idxSettings.getNumberOfShards(), - idxSettings.getNumberOfReplicas(), - idxSettings.isShadowReplicaIndex() ? "s" : ""); - - final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry); - pluginsService.onIndexModule(indexModule); - for (IndexEventListener listener : builtInListeners) { - indexModule.addIndexEventListener(listener); - } + List finalListeners = new ArrayList<>(builtInListeners); final IndexEventListener onStoreClose = new IndexEventListener() { @Override public void onStoreClosed(ShardId shardId) { indicesQueryCache.onClose(shardId); } }; - indexModule.addIndexEventListener(onStoreClose); - indexModule.addIndexEventListener(oldShardsStats); - final IndexEventListener listener = indexModule.freeze(); - listener.beforeIndexCreated(index, idxSettings.getSettings()); - final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingMemoryController); + finalListeners.add(onStoreClose); + finalListeners.add(oldShardsStats); + final IndexService indexService = createIndexService("create index", nodeServicesProvider, indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController); boolean success = false; try { - assert indexService.getIndexEventListener() == listener; - listener.afterIndexCreated(indexService); + indexService.getIndexEventListener().afterIndexCreated(indexService); indices = newMapBuilder(indices).put(index.getUUID(), indexService).immutableMap(); success = true; return indexService; @@ -370,7 +362,54 @@ public void onStoreClosed(ShardId shardId) { indexService.close("plugins_failed", true); } } + } + /** + * This creates a new IndexService without registering it + */ + private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException { + final Index index = indexMetaData.getIndex(); + final Predicate indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state()); + final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting); + logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]", + indexMetaData.getIndex(), + idxSettings.getNumberOfShards(), + idxSettings.getNumberOfReplicas(), + idxSettings.isShadowReplicaIndex() ? "s" : "", reason); + + final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry); + pluginsService.onIndexModule(indexModule); + for (IndexEventListener listener : builtInListeners) { + indexModule.addIndexEventListener(listener); + } + final IndexEventListener listener = indexModule.freeze(); + listener.beforeIndexCreated(index, idxSettings.getSettings()); + return indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingOperationListeners); + } + + /** + * This method verifies that the given {@link IndexMetaData} holds sane values to create an {@link IndexService}. This method will throw an + * exception if the creation fails. The created {@link IndexService} will not be registered and will be closed immediately. + */ + public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData) throws IOException { + final List closeables = new ArrayList<>(); + try { + IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {}); + closeables.add(indicesFieldDataCache); + IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings); + closeables.add(indicesQueryCache); + // this will also fail if some plugin fails etc. which is nice since we can verify that early + final IndexService service = createIndexService("metadata verification", nodeServicesProvider, + metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList()); + for (ObjectCursor typeMapping : metaData.getMappings().values()) { + // don't apply the default mapping, it has been applied when the mapping was created + service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(), + MapperService.MergeReason.MAPPING_RECOVERY, true); + } + closeables.add(() -> service.close("metadata verification", false)); + } finally { + IOUtils.close(closeables); + } } /** diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index d0d9b227948aa..af2894833cb7d 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -19,9 +19,13 @@ package org.elasticsearch.gateway; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.client.Requests; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -31,7 +35,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.indices.IndexClosedException; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -40,6 +48,7 @@ import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; @@ -364,4 +373,123 @@ public Settings onNodeStopped(String nodeName) throws Exception { logger.info("--> verify the doc is there"); assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); } + + /** + * This test really tests worst case scenario where we have a broken setting or any setting that prevents an index from being + * allocated in our metadata that we recover. In that case we now have the ability to check the index on local recovery from disk + * if it is sane and if we can successfully create an IndexService. This also includes plugins etc. + */ + public void testRecoverBrokenIndexMetadata() throws Exception { + logger.info("--> starting one node"); + internalCluster().startNode(); + logger.info("--> indexing a simple document"); + client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet(); + logger.info("--> waiting for green status"); + if (usually()) { + ensureYellow(); + } else { + internalCluster().startNode(); + client().admin().cluster() + .health(Requests.clusterHealthRequest() + .waitForGreenStatus() + .waitForEvents(Priority.LANGUID) + .waitForRelocatingShards(0).waitForNodes("2")).actionGet(); + } + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetaData metaData = state.getMetaData().index("test"); + for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) { + IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(Settings.builder().put(metaData.getSettings()) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id) + // this is invalid but should be archived + .put("index.similarity.BM25.type", "classic") + // this one is not validated ahead of time and breaks allocation + .put("index.analysis.filter.myCollator.type", "icu_collation") + ).build(); + IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex())); + } + internalCluster().fullRestart(); + // ensureGreen(closedIndex) waits for the index to show up in the metadata + // this is crucial otherwise the state call below might not contain the index yet + ensureGreen(metaData.getIndex().getName()); + state = client().admin().cluster().prepareState().get().getState(); + assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState()); + assertEquals("classic", state.getMetaData().index(metaData.getIndex()).getSettings().get("archived.index.similarity.BM25.type")); + // try to open it with the broken setting - fail again! + ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get()); + assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex()); + assertNotNull(ex.getCause()); + assertEquals(IllegalArgumentException.class, ex.getCause().getClass()); + assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]"); + + client().admin().indices().prepareUpdateSettings() + .setSettings(Settings.builder().putNull("index.analysis.filter.myCollator.type")).get(); + client().admin().indices().prepareOpen("test").get(); + ensureYellow(); + logger.info("--> verify 1 doc in the index"); + assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); + } + + /** + * This test really tests worst case scenario where we have a missing analyzer setting. + * In that case we now have the ability to check the index on local recovery from disk + * if it is sane and if we can successfully create an IndexService. + * This also includes plugins etc. + */ + public void testRecoverMissingAnalyzer() throws Exception { + logger.info("--> starting one node"); + internalCluster().startNode(); + prepareCreate("test").setSettings(Settings.builder() + .put("index.analysis.analyzer.test.tokenizer", "keyword") + .put("index.number_of_shards", "1")) + .addMapping("type1", "{\n" + + " \"type1\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\",\n" + + " \"analyzer\": \"test\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }}").get(); + logger.info("--> indexing a simple document"); + client().prepareIndex("test", "type1", "1").setSource("field1", "value one").setRefresh(true).execute().actionGet(); + logger.info("--> waiting for green status"); + if (usually()) { + ensureYellow(); + } else { + internalCluster().startNode(); + client().admin().cluster() + .health(Requests.clusterHealthRequest() + .waitForGreenStatus() + .waitForEvents(Priority.LANGUID) + .waitForRelocatingShards(0).waitForNodes("2")).actionGet(); + } + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetaData metaData = state.getMetaData().index("test"); + for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) { + IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings() + .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build(); + IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex())); + } + internalCluster().fullRestart(); + // ensureGreen(closedIndex) waits for the index to show up in the metadata + // this is crucial otherwise the state call below might not contain the index yet + ensureGreen(metaData.getIndex().getName()); + state = client().admin().cluster().prepareState().get().getState(); + assertEquals(IndexMetaData.State.CLOSE, state.getMetaData().index(metaData.getIndex()).getState()); + + // try to open it with the broken setting - fail again! + ElasticsearchException ex = expectThrows(ElasticsearchException.class, () -> client().admin().indices().prepareOpen("test").get()); + assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex()); + assertNotNull(ex.getCause()); + assertEquals(MapperParsingException.class, ex.getCause().getClass()); + assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]"); + + client().admin().indices().prepareUpdateSettings() + .setSettings(Settings.builder().put("index.analysis.analyzer.test.tokenizer", "keyword")).get(); + client().admin().indices().prepareOpen("test").get(); + ensureYellow(); + logger.info("--> verify 1 doc in the index"); + assertHitCount(client().prepareSearch().setQuery(matchQuery("field1", "value one")).get(), 1L); + } } diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index bf9921a2e23eb..3a3f5f67e776a 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -40,8 +40,7 @@ private GatewayService createService(Settings.Builder settings) { .put("http.enabled", "false") .put("discovery.type", "local") .put(settings.build()).build(), - null, clusterService, null, null, null, null, new NoopDiscovery()); - + null, clusterService, null, null, null, null, new NoopDiscovery(), null, null); } public void testDefaultRecoverAfterTime() throws IOException {