From 2efffab9419d762fbd446c66bbec99b14b7c206f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 22 Nov 2018 20:30:08 +0100 Subject: [PATCH 01/62] SNAPSHOT: Keep SnapshotsInProgress State in Sync with Routing Table (#35710) --- .../routing/RoutingChangesObserver.java | 49 ++++++ .../routing/allocation/AllocationService.java | 19 ++- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ++++++++++++++++++ .../DedicatedClusterSnapshotRestoreIT.java | 46 ++++++ .../BusyMasterServiceDisruption.java | 89 +++++++++++ 7 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 883b4c22f7fc0..0c0d3c5a099f9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,6 +19,8 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.index.shard.ShardId; + /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -211,4 +213,51 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } + + abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { + + @Override + public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { + onChanged(unassignedShard.shardId()); + } + + @Override + public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { + onChanged(initializingShard.shardId()); + } + @Override + public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { + onChanged(startedShard.shardId()); + } + @Override + public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { + onChanged(unassignedShard.shardId()); + } + @Override + public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { + onChanged(failedShard.shardId()); + } + @Override + public void relocationCompleted(ShardRouting removedRelocationSource) { + onChanged(removedRelocationSource.shardId()); + } + @Override + public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { + onChanged(removedReplicaRelocationSource.shardId()); + } + @Override + public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { + onChanged(startedPrimaryShard.shardId()); + } + @Override + public void replicaPromoted(ShardRouting replicaShard) { + onChanged(replicaShard.shardId()); + } + @Override + public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { + onChanged(oldReplica.shardId()); + } + + protected abstract void onChanged(ShardId shardId); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 59f43a193ddc8..389eec6ed5a87 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -136,15 +137,29 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); + ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); - newStateBuilder.customs(customsBuilder.build()); } } + final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + SnapshotsInProgress updatedSnapshotsInProgress = + allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); + if (updatedSnapshotsInProgress != snapshotsInProgress) { + if (customsBuilder == null) { + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + } + customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); + } + } + if (customsBuilder != null) { + newStateBuilder.customs(customsBuilder.build()); + } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index e0be712a230c7..f178ca18845ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -76,11 +78,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); + private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater ); - /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -251,6 +253,14 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } + /** + * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes + */ + public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, + RoutingTable newRoutingTable) { + return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); + } + /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index c505a0a28b65d..b4d7065f38495 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -313,7 +313,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to starts + // We have new shards to start if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 4c1b84b0f2233..43ea03ee1a5df 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -45,6 +46,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -664,12 +666,28 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); + // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here + assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } + private boolean assertConsistency(ClusterState state) { + SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + assert snapshotsInProgress == updateWithRoutingTable( + snapshotsInProgress.entries().stream().flatMap(entry -> { + Iterable iterable = () -> entry.shards().keysIt(); + return StreamSupport.stream(iterable.spliterator(), false); + }).collect(Collectors.toSet()), + snapshotsInProgress, state.routingTable() + ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; + } + return true; + } + /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1551,6 +1569,128 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } + public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { + + private final Set shardChanges = new HashSet<>(); + + public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { + return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); + } + + @Override + protected void onChanged(ShardId shardId) { + shardChanges.add(shardId); + } + } + + private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, + RoutingTable newRoutingTable) { + if (oldSnapshot == null || shardIds.isEmpty()) { + return oldSnapshot; + } + List entries = new ArrayList<>(); + boolean snapshotsInProgressChanged = false; + for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { + ImmutableOpenMap.Builder shardsBuilder = null; + for (ShardId shardId : shardIds) { + final ImmutableOpenMap shards = entry.shards(); + final ShardSnapshotStatus currentStatus = shards.get(shardId); + if (currentStatus != null && currentStatus.state().completed() == false) { + IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); + assert routingTable != null; + final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) + .map(IndexShardRoutingTable::primaryShard) + .map( + primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) + ) + .orElse(failedStatus(null, "missing shard")); + if (newStatus != currentStatus) { + if (shardsBuilder == null) { + shardsBuilder = ImmutableOpenMap.builder(shards); + } + shardsBuilder.put(shardId, newStatus); + } + } + } + if (shardsBuilder == null) { + entries.add(entry); + } else { + snapshotsInProgressChanged = true; + ImmutableOpenMap shards = shardsBuilder.build(); + entries.add( + new SnapshotsInProgress.Entry( + entry, + completed(shards.values()) ? State.SUCCESS : entry.state(), + shards + ) + ); + } + } + if (snapshotsInProgressChanged) { + return new SnapshotsInProgress(entries); + } + return oldSnapshot; + } + + private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, + final ShardRouting primaryShardRouting) { + final State currentState = currentStatus.state(); + final ShardSnapshotStatus newStatus; + if (primaryShardRouting.active() == false) { + if (primaryShardRouting.initializing() && currentState == State.WAITING) { + newStatus = currentStatus; + } else { + newStatus = failedStatus( + primaryShardRouting.currentNodeId(), + primaryShardRouting.unassignedInfo().getReason().toString() + ); + } + } else if (primaryShardRouting.started()) { + switch (currentState) { + case WAITING: + newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); + break; + case INIT: { + String currentNodeId = currentStatus.nodeId(); + assert currentNodeId != null; + if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + } + case ABORTED: + String currentNodeId = currentStatus.nodeId(); + if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + default: + newStatus = currentStatus; + break; + } + } else { + assert primaryShardRouting.relocating(); + if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { + newStatus = failedStatus(currentStatus.nodeId()); + } else { + newStatus = currentStatus; + } + } + return newStatus; + } + + private static ShardSnapshotStatus failedStatus(String nodeId) { + return failedStatus(nodeId, "shard failed"); + } + + private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { + return new ShardSnapshotStatus(nodeId, State.FAILED, reason); + } + /** * Snapshot creation request */ diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 3347e2b8ee453..66a133c6abeb4 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -78,6 +78,8 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.TestCustomMetaData; +import org.elasticsearch.test.disruption.BusyMasterServiceDisruption; +import org.elasticsearch.test.disruption.ServiceDisruptionScheme; import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; @@ -1151,6 +1153,50 @@ public void testSnapshotTotalAndIncrementalSizes() throws IOException { assertThat(anotherStats.getTotalSize(), is(snapshot1FileSize)); } + public void testDataNodeRestartWithBusyMasterDuringSnapshot() throws Exception { + logger.info("--> starting a master node and two data nodes"); + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNodes(2); + logger.info("--> creating repository"); + assertAcked(client().admin().cluster().preparePutRepository("test-repo") + .setType("mock").setSettings(Settings.builder() + .put("location", randomRepoPath()) + .put("compress", randomBoolean()) + .put("max_snapshot_bytes_per_sec", "1000b") + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + assertAcked(prepareCreate("test-idx", 0, Settings.builder() + .put("number_of_shards", 5).put("number_of_replicas", 0))); + ensureGreen(); + logger.info("--> indexing some data"); + final int numdocs = randomIntBetween(50, 100); + IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; + for (int i = 0; i < builders.length; i++) { + builders[i] = client().prepareIndex("test-idx", "type1", + Integer.toString(i)).setSource("field1", "bar " + i); + } + indexRandom(true, builders); + flushAndRefresh(); + final String dataNode = blockNodeWithIndex("test-repo", "test-idx"); + logger.info("--> snapshot"); + client(internalCluster().getMasterName()).admin().cluster() + .prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get(); + ServiceDisruptionScheme disruption = new BusyMasterServiceDisruption(random(), Priority.HIGH); + setDisruptionScheme(disruption); + disruption.startDisrupting(); + logger.info("--> restarting data node, which should cause primary shards to be failed"); + internalCluster().restartNode(dataNode, InternalTestCluster.EMPTY_CALLBACK); + unblockNode("test-repo", dataNode); + disruption.stopDisrupting(); + // check that snapshot completes + assertBusy(() -> { + GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster() + .prepareGetSnapshots("test-repo").setSnapshots("test-snap").setIgnoreUnavailable(true).get(); + assertEquals(1, snapshotsStatusResponse.getSnapshots().size()); + SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0); + assertTrue(snapshotInfo.state().toString(), snapshotInfo.state().completed()); + }, 30, TimeUnit.SECONDS); + } + private long calculateTotalFilesSize(List files) { return files.stream().mapToLong(f -> { try { diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java new file mode 100644 index 0000000000000..3621cba1e7992 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BusyMasterServiceDisruption.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.disruption; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.InternalTestCluster; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BusyMasterServiceDisruption extends SingleNodeDisruption { + private final AtomicBoolean active = new AtomicBoolean(); + private final Priority priority; + + public BusyMasterServiceDisruption(Random random, Priority priority) { + super(random); + this.priority = priority; + } + + @Override + public void startDisrupting() { + disruptedNode = cluster.getMasterName(); + final String disruptionNodeCopy = disruptedNode; + if (disruptionNodeCopy == null) { + return; + } + ClusterService clusterService = cluster.getInstance(ClusterService.class, disruptionNodeCopy); + if (clusterService == null) { + return; + } + logger.info("making master service busy on node [{}] at priority [{}]", disruptionNodeCopy, priority); + active.set(true); + submitTask(clusterService); + } + + private void submitTask(ClusterService clusterService) { + clusterService.getMasterService().submitStateUpdateTask( + "service_disruption_block", + new ClusterStateUpdateTask(priority) { + @Override + public ClusterState execute(ClusterState currentState) { + if (active.get()) { + submitTask(clusterService); + } + return currentState; + } + + @Override + public void onFailure(String source, Exception e) { + logger.error("unexpected error during disruption", e); + } + } + ); + } + + @Override + public void stopDisrupting() { + active.set(false); + } + + @Override + public void removeAndEnsureHealthy(InternalTestCluster cluster) { + removeFromCluster(cluster); + } + + @Override + public TimeValue expectedTimeToHeal() { + return TimeValue.timeValueMinutes(0); + } +} From 715ae9f98fff086b7b18c0455f5136396bb027b0 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 22 Nov 2018 11:41:00 +0100 Subject: [PATCH 02/62] SQL: Implement NVL(expr1, expr2) (#35794) Add NVL as alias to IFNULL as they have the same behaviour. Add basic tests and docs. Closes: #35782 --- .../sql/functions/conditional.asciidoc | 37 +++++++++++++++++++ .../qa/src/main/resources/command.csv-spec | 3 +- .../sql/qa/src/main/resources/docs.csv-spec | 25 ++++++++++++- .../expression/function/FunctionRegistry.java | 2 +- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/reference/sql/functions/conditional.asciidoc b/docs/reference/sql/functions/conditional.asciidoc index 944b42bfdcd8d..75782b894af87 100644 --- a/docs/reference/sql/functions/conditional.asciidoc +++ b/docs/reference/sql/functions/conditional.asciidoc @@ -118,3 +118,40 @@ include-tagged::{sql-specs}/docs.csv-spec[isNullReturnFirst] ---- include-tagged::{sql-specs}/docs.csv-spec[isNullReturnSecond] ---- + + +[[sql-functions-conditional-nvl]] +==== `NVL` + +.Synopsis +[source, sql] +---- +NVL ( expression<1>, expression<2> ) +---- + +*Input*: + +<1> 1st expression + +<2> 2nd expression + + +*Output*: 2nd expression if 1st expression is null, otherwise 1st expression. + +.Description + +Variant of <> with only two arguments. +Returns the first of its arguments that is not null. +If all arguments are null, then it returns `null`. + + + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs.csv-spec[nvlReturnFirst] +---- + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs.csv-spec[nvlReturnSecond] +---- diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec index 7cfeb0e362806..62f418dce2bd7 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec @@ -22,8 +22,9 @@ VAR_POP |AGGREGATE COALESCE |CONDITIONAL IFNULL |CONDITIONAL ISNULL |CONDITIONAL +NVL |CONDITIONAL DAY |SCALAR -DAYNAME |SCALAR +DAYNAME |SCALAR DAYOFMONTH |SCALAR DAYOFWEEK |SCALAR DAYOFYEAR |SCALAR diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec index 928cc3b38c63b..c8410e6c9ea30 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec @@ -199,8 +199,9 @@ VAR_POP |AGGREGATE COALESCE |CONDITIONAL IFNULL |CONDITIONAL ISNULL |CONDITIONAL +NVL |CONDITIONAL DAY |SCALAR -DAYNAME |SCALAR +DAYNAME |SCALAR DAYOFMONTH |SCALAR DAYOFWEEK |SCALAR DAYOFYEAR |SCALAR @@ -1555,7 +1556,6 @@ search // end::ifNullReturnSecond ; - isNullReturnFirst // tag::isNullReturnFirst SELECT ISNULL('elastic', null) AS "isnull"; @@ -1576,3 +1576,24 @@ SELECT ISNULL(null, 'search') AS "isnull"; search // end::isNullReturnSecond ; + +nvlReturnFirst +// tag::nvlReturnFirst +SELECT NVL('elastic', null) AS "nvl"; + + nvl +--------------- +elastic +// end::nvlReturnFirst +; + + +nvlReturnSecond +// tag::nvlReturnSecond +SELECT NVL(null, 'search') AS "nvl"; + + nvl +--------------- +search +// end::nvlReturnSecond +; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java index ef8e55de6d5b1..077b5cab78f51 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java @@ -146,7 +146,7 @@ private void defineDefaultFunctions() { // Scalar functions // conditional addToMap(def(Coalesce.class, Coalesce::new)); - addToMap(def(IFNull.class, IFNull::new, "ISNULL")); + addToMap(def(IFNull.class, IFNull::new, "ISNULL", "NVL")); // Date addToMap(def(DayName.class, DayName::new, "DAYNAME"), def(DayOfMonth.class, DayOfMonth::new, "DAYOFMONTH", "DAY", "DOM"), From 3f794764df7499323b6ef6b728f883e1953efdbc Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 22 Nov 2018 06:08:48 -0500 Subject: [PATCH 03/62] Forbid negative scores in functon_score query (#35709) * Forbid negative scores in functon_score query - Throw an exception when scores are negative in field_value_factor function - Throw an exception when scores are negative in script_score function Relates to #33309 --- .../migration/migrate_7_0/search.asciidoc | 10 ++++- .../query-dsl/function-score-query.asciidoc | 6 +++ .../rest-api-spec/test/painless/30_search.yml | 38 +++++++++++++++++++ .../function/FieldValueFactorFunction.java | 4 ++ .../search/function/FunctionScoreQuery.java | 2 +- .../search/function/ScriptScoreFunction.java | 3 ++ .../functionscore/FunctionScoreTests.java | 14 +++++++ 7 files changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc index efd2d2c271a8d..2576a0e5136f3 100644 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ b/docs/reference/migration/migrate_7_0/search.asciidoc @@ -148,4 +148,12 @@ instead which is a more appropriate value for a scenario where scores are not av ==== Negative boosts are not allowed Setting a negative `boost` in a query, deprecated in 6x, are not allowed in this version. -To deboost a specific query you can use a `boost` comprise between 0 and 1. \ No newline at end of file +To deboost a specific query you can use a `boost` comprise between 0 and 1. + +[float] +==== Negative scores are not allowed in Function Score Query + +Negative scores in the Function Score Query are deprecated in 6.x, and are +not allowed in this version. If a negative score is produced as a result +of computation (e.g. in `script_score` or `field_value_factor` functions), +an error will be thrown. \ No newline at end of file diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc index 7c5ca95623e83..71fa61ee085e3 100644 --- a/docs/reference/query-dsl/function-score-query.asciidoc +++ b/docs/reference/query-dsl/function-score-query.asciidoc @@ -153,6 +153,9 @@ GET /_search // CONSOLE // TEST[setup:twitter] +NOTE: Scores produced by the `script_score` function must be non-negative, +otherwise an error will be thrown. + On top of the different scripting field values and expression, the `_score` script parameter can be used to retrieve the score based on the wrapped query. @@ -324,6 +327,9 @@ There are a number of options for the `field_value_factor` function: values of the field with a range filter to avoid this, or use `log1p` and `ln1p`. + NOTE: Scores produced by the `field_value_score` function must be non-negative, + otherwise an error will be thrown. + [[function-decay]] ==== Decay functions diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml index 0c0e980d95a6f..8decbffe84d57 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml @@ -442,3 +442,41 @@ - match: { error.root_cause.0.reason: "Iterable object is self-referencing itself (ScriptBytesValues value)" } - match: { error.type: "search_phase_execution_exception" } - match: { error.reason: "all shards failed" } + + +--- + +"Exception on negative score": + - skip: + version: " - 6.99.99" + reason: "check on negative scores was added from 7.0.0 on" + + - do: + index: + index: test + type: test + id: 1 + body: { "test": "value beck", "num1": 1.0 } + - do: + indices.refresh: {} + + - do: + catch: bad_request + search: + index: test + body: + query: + function_score: + query: + term: + test: value + "functions": [{ + "script_score": { + "script": { + "lang": "painless", + "source": "doc['num1'].value - 10.0" + } + } + }] + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "script score function must not produce negative scores, but got: [-9.0]"} diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java index fb5a82bc098e2..580ea97de1526 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FieldValueFactorFunction.java @@ -83,6 +83,10 @@ public double score(int docId, float subQueryScore) throws IOException { } double val = value * boostFactor; double result = modifier.apply(val); + if (result < 0f) { + throw new IllegalArgumentException("field value function must not produce negative scores, but got: [" + + result + "] for field value: [" + value + "]"); + } return result; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java index fe0901c6cdb3f..95fbed64a25f5 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java @@ -382,7 +382,7 @@ public float score() throws IOException { } double factor = computeScore(docId, subQueryScore); float finalScore = scoreCombiner.combine(subQueryScore, factor, maxBoost); - if (finalScore == Float.NEGATIVE_INFINITY || Float.isNaN(finalScore)) { + if (finalScore < 0f || Float.isNaN(finalScore)) { /* These scores are invalid for score based {@link org.apache.lucene.search.TopDocsCollector}s. See {@link org.apache.lucene.search.TopScoreDocCollector} for details. diff --git a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java index 3893572aa4494..8e51bc5951d59 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java @@ -69,6 +69,9 @@ public double score(int docId, float subQueryScore) throws IOException { scorer.docid = docId; scorer.score = subQueryScore; double result = leafScript.execute(); + if (result < 0f) { + throw new IllegalArgumentException("script score function must not produce negative scores, but got: [" + result + "]"); + } return result; } diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index 9b11017e5980e..fa0a372a2adb8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -824,6 +824,20 @@ public void testWithInvalidScores() { assertThat(exc.getMessage(), containsString("function score query returned an invalid score: " + Float.NEGATIVE_INFINITY)); } + + public void testExceptionOnNegativeScores() { + IndexSearcher localSearcher = new IndexSearcher(reader); + TermQuery termQuery = new TermQuery(new Term(FIELD, "out")); + + // test that field_value_factor function throws an exception on negative scores + FieldValueFactorFunction.Modifier modifier = FieldValueFactorFunction.Modifier.NONE; + final ScoreFunction fvfFunction = new FieldValueFactorFunction(FIELD, -10, modifier, 1.0, new IndexNumericFieldDataStub()); + FunctionScoreQuery fsQuery1 = + new FunctionScoreQuery(termQuery, fvfFunction, CombineFunction.REPLACE, null, Float.POSITIVE_INFINITY); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> localSearcher.search(fsQuery1, 1)); + assertThat(exc.getMessage(), containsString("field value function must not produce negative scores")); + } + private static class DummyScoreFunction extends ScoreFunction { protected DummyScoreFunction(CombineFunction scoreCombiner) { super(scoreCombiner); From 9293189c34fa126b6914d4b853d21129c2dab83c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 22 Nov 2018 12:13:09 +0100 Subject: [PATCH 04/62] Revert "Revert "[RCI] Check blocks while having index shard permit in TransportReplicationAction (#35332)"" This reverts commit d3d7c01 --- .../TransportReplicationAction.java | 137 +++++----- .../TransportReplicationActionTests.java | 233 ++++++++++++------ 2 files changed, 234 insertions(+), 136 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index e030fa4f15190..38972a7f77462 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -235,9 +235,39 @@ protected TransportRequestOptions transportOptions(Settings settings) { return TransportRequestOptions.EMPTY; } + private String concreteIndex(final ClusterState state, final ReplicationRequest request) { + return resolveIndex() ? indexNameExpressionResolver.concreteSingleIndex(state, request).getName() : request.index(); + } + + private ClusterBlockException blockExceptions(final ClusterState state, final String indexName) { + ClusterBlockLevel globalBlockLevel = globalBlockLevel(); + if (globalBlockLevel != null) { + ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel); + if (blockException != null) { + return blockException; + } + } + ClusterBlockLevel indexBlockLevel = indexBlockLevel(); + if (indexBlockLevel != null) { + ClusterBlockException blockException = state.blocks().indexBlockedException(indexBlockLevel, indexName); + if (blockException != null) { + return blockException; + } + } + return null; + } + protected boolean retryPrimaryException(final Throwable e) { return e.getClass() == ReplicationOperation.RetryOnPrimaryException.class - || TransportActions.isShardNotAvailableException(e); + || TransportActions.isShardNotAvailableException(e) + || isRetryableClusterBlockException(e); + } + + boolean isRetryableClusterBlockException(final Throwable e) { + if (e instanceof ClusterBlockException) { + return ((ClusterBlockException) e).retryable(); + } + return false; } protected class OperationTransportHandler implements TransportRequestHandler { @@ -310,6 +340,15 @@ protected void doRun() throws Exception { @Override public void onResponse(PrimaryShardReference primaryShardReference) { try { + final ClusterState clusterState = clusterService.state(); + final IndexMetaData indexMetaData = clusterState.metaData().getIndexSafe(primaryShardReference.routingEntry().index()); + + final ClusterBlockException blockException = blockExceptions(clusterState, indexMetaData.getIndex().getName()); + if (blockException != null) { + logger.trace("cluster is blocked, action failed on primary", blockException); + throw blockException; + } + if (primaryShardReference.isRelocated()) { primaryShardReference.close(); // release shard operation lock as soon as possible setPhase(replicationTask, "primary_delegation"); @@ -323,7 +362,7 @@ public void onResponse(PrimaryShardReference primaryShardReference) { response.readFrom(in); return response; }; - DiscoveryNode relocatingNode = clusterService.state().nodes().get(primary.relocatingNodeId()); + DiscoveryNode relocatingNode = clusterState.nodes().get(primary.relocatingNodeId()); transportService.sendRequest(relocatingNode, transportPrimaryAction, new ConcreteShardRequest<>(request, primary.allocationId().getRelocationId(), primaryTerm), transportOptions, @@ -696,35 +735,42 @@ public void onFailure(Exception e) { protected void doRun() { setPhase(task, "routing"); final ClusterState state = observer.setAndGetObservedState(); - if (handleBlockExceptions(state)) { - return; - } - - // request does not have a shardId yet, we need to pass the concrete index to resolve shardId - final String concreteIndex = concreteIndex(state); - final IndexMetaData indexMetaData = state.metaData().index(concreteIndex); - if (indexMetaData == null) { - retry(new IndexNotFoundException(concreteIndex)); - return; - } - if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { - throw new IndexClosedException(indexMetaData.getIndex()); - } + final String concreteIndex = concreteIndex(state, request); + final ClusterBlockException blockException = blockExceptions(state, concreteIndex); + if (blockException != null) { + if (blockException.retryable()) { + logger.trace("cluster is blocked, scheduling a retry", blockException); + retry(blockException); + } else { + finishAsFailed(blockException); + } + } else { + // request does not have a shardId yet, we need to pass the concrete index to resolve shardId + final IndexMetaData indexMetaData = state.metaData().index(concreteIndex); + if (indexMetaData == null) { + retry(new IndexNotFoundException(concreteIndex)); + return; + } + if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { + throw new IndexClosedException(indexMetaData.getIndex()); + } - // resolve all derived request fields, so we can route and apply it - resolveRequest(indexMetaData, request); - assert request.shardId() != null : "request shardId must be set in resolveRequest"; - assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "request waitForActiveShards must be set in resolveRequest"; + // resolve all derived request fields, so we can route and apply it + resolveRequest(indexMetaData, request); + assert request.shardId() != null : "request shardId must be set in resolveRequest"; + assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : + "request waitForActiveShards must be set in resolveRequest"; - final ShardRouting primary = primary(state); - if (retryIfUnavailable(state, primary)) { - return; - } - final DiscoveryNode node = state.nodes().get(primary.currentNodeId()); - if (primary.currentNodeId().equals(state.nodes().getLocalNodeId())) { - performLocalAction(state, primary, node, indexMetaData); - } else { - performRemoteAction(state, primary, node); + final ShardRouting primary = primary(state); + if (retryIfUnavailable(state, primary)) { + return; + } + final DiscoveryNode node = state.nodes().get(primary.currentNodeId()); + if (primary.currentNodeId().equals(state.nodes().getLocalNodeId())) { + performLocalAction(state, primary, node, indexMetaData); + } else { + performRemoteAction(state, primary, node); + } } } @@ -776,44 +822,11 @@ private boolean retryIfUnavailable(ClusterState state, ShardRouting primary) { return false; } - private String concreteIndex(ClusterState state) { - return resolveIndex() ? indexNameExpressionResolver.concreteSingleIndex(state, request).getName() : request.index(); - } - private ShardRouting primary(ClusterState state) { IndexShardRoutingTable indexShard = state.getRoutingTable().shardRoutingTable(request.shardId()); return indexShard.primaryShard(); } - private boolean handleBlockExceptions(ClusterState state) { - ClusterBlockLevel globalBlockLevel = globalBlockLevel(); - if (globalBlockLevel != null) { - ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel); - if (blockException != null) { - handleBlockException(blockException); - return true; - } - } - ClusterBlockLevel indexBlockLevel = indexBlockLevel(); - if (indexBlockLevel != null) { - ClusterBlockException blockException = state.blocks().indexBlockedException(indexBlockLevel, concreteIndex(state)); - if (blockException != null) { - handleBlockException(blockException); - return true; - } - } - return false; - } - - private void handleBlockException(ClusterBlockException blockException) { - if (blockException.retryable()) { - logger.trace("cluster is blocked, scheduling a retry", blockException); - retry(blockException); - } else { - finishAsFailed(blockException); - } - } - private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction, final TransportRequest requestToPerform) { transportService.sendRequest(node, action, requestToPerform, transportOptions, new TransportResponseHandler() { diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index aeda5f1c3fa80..c8c40a7f5841a 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -89,6 +89,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -100,6 +101,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static java.util.Collections.singleton; import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state; import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS; @@ -108,9 +110,11 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.core.Is.is; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyLong; @@ -182,70 +186,157 @@ public static void afterClass() { threadPool = null; } - void assertListenerThrows(String msg, PlainActionFuture listener, Class klass) throws InterruptedException { - try { - listener.get(); - fail(msg); - } catch (ExecutionException ex) { - assertThat(ex.getCause(), instanceOf(klass)); + private T assertListenerThrows(String msg, PlainActionFuture listener, Class klass) { + ExecutionException exception = expectThrows(ExecutionException.class, msg, listener::get); + assertThat(exception.getCause(), instanceOf(klass)); + @SuppressWarnings("unchecked") + final T cause = (T) exception.getCause(); + return cause; + } + + private void setStateWithBlock(final ClusterService clusterService, final ClusterBlock block, final boolean globalBlock) { + final ClusterBlocks.Builder blocks = ClusterBlocks.builder(); + if (globalBlock) { + blocks.addGlobalBlock(block); + } else { + blocks.addIndexBlock("index", block); } + setState(clusterService, ClusterState.builder(clusterService.state()).blocks(blocks).build()); } - public void testBlocks() throws ExecutionException, InterruptedException { - Request request = new Request(); - PlainActionFuture listener = new PlainActionFuture<>(); - ReplicationTask task = maybeTask(); - TestAction action = new TestAction(Settings.EMPTY, "internal:testActionWithBlocks", - transportService, clusterService, shardStateAction, threadPool) { + public void testBlocksInReroutePhase() throws Exception { + final ClusterBlock nonRetryableBlock = + new ClusterBlock(1, "non retryable", false, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL); + final ClusterBlock retryableBlock = + new ClusterBlock(1, "retryable", true, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL); + + final boolean globalBlock = randomBoolean(); + final TestAction action = new TestAction(Settings.EMPTY, "internal:testActionWithBlocks", + transportService, clusterService, shardStateAction, threadPool) { @Override protected ClusterBlockLevel globalBlockLevel() { - return ClusterBlockLevel.WRITE; + return globalBlock ? ClusterBlockLevel.WRITE : null; + } + + @Override + protected ClusterBlockLevel indexBlockLevel() { + return globalBlock == false ? ClusterBlockLevel.WRITE : null; } }; - ClusterBlocks.Builder block = ClusterBlocks.builder().addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, - false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); - setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block)); - TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); - reroutePhase.run(); - assertListenerThrows("primary phase should fail operation", listener, ClusterBlockException.class); - assertPhase(task, "failed"); + setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary("index", true, 0)); - block = ClusterBlocks.builder() - .addGlobalBlock(new ClusterBlock(1, "retryable", true, true, false, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); - setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block)); - listener = new PlainActionFuture<>(); - reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener); - reroutePhase.run(); - assertListenerThrows("failed to timeout on retryable block", listener, ClusterBlockException.class); - assertPhase(task, "failed"); - assertFalse(request.isRetrySet.get()); + { + setStateWithBlock(clusterService, nonRetryableBlock, globalBlock); - listener = new PlainActionFuture<>(); - reroutePhase = action.new ReroutePhase(task, request = new Request(), listener); - reroutePhase.run(); - assertFalse("primary phase should wait on retryable block", listener.isDone()); - assertPhase(task, "waiting_for_retry"); - assertTrue(request.isRetrySet.get()); + Request request = globalBlock ? new Request() : new Request().index("index"); + PlainActionFuture listener = new PlainActionFuture<>(); + ReplicationTask task = maybeTask(); + + TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + reroutePhase.run(); + + ClusterBlockException exception = + assertListenerThrows("primary action should fail operation", listener, ClusterBlockException.class); + assertThat(((ClusterBlockException) exception.unwrapCause()).blocks().iterator().next(), is(nonRetryableBlock)); + assertPhase(task, "failed"); + } + { + setStateWithBlock(clusterService, retryableBlock, globalBlock); + + Request requestWithTimeout = (globalBlock ? new Request() : new Request().index("index")).timeout("5ms"); + PlainActionFuture listener = new PlainActionFuture<>(); + ReplicationTask task = maybeTask(); + + TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, requestWithTimeout, listener); + reroutePhase.run(); + + ClusterBlockException exception = + assertListenerThrows("failed to timeout on retryable block", listener, ClusterBlockException.class); + assertThat(((ClusterBlockException) exception.unwrapCause()).blocks().iterator().next(), is(retryableBlock)); + assertPhase(task, "failed"); + assertTrue(requestWithTimeout.isRetrySet.get()); + } + { + setStateWithBlock(clusterService, retryableBlock, globalBlock); + + Request request = globalBlock ? new Request() : new Request().index("index"); + PlainActionFuture listener = new PlainActionFuture<>(); + ReplicationTask task = maybeTask(); + + TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + reroutePhase.run(); + + assertFalse("primary phase should wait on retryable block", listener.isDone()); + assertPhase(task, "waiting_for_retry"); + assertTrue(request.isRetrySet.get()); + + setStateWithBlock(clusterService, nonRetryableBlock, globalBlock); + + ClusterBlockException exception = assertListenerThrows("primary phase should fail operation when moving from a retryable " + + "block to a non-retryable one", listener, ClusterBlockException.class); + assertThat(((ClusterBlockException) exception.unwrapCause()).blocks().iterator().next(), is(nonRetryableBlock)); + assertIndexShardUninitialized(); + } + { + Request requestWithTimeout = new Request().index("unknown").setShardId(new ShardId("unknown", "_na_", 0)).timeout("5ms"); + PlainActionFuture listener = new PlainActionFuture<>(); + ReplicationTask task = maybeTask(); + + TestAction testActionWithNoBlocks = new TestAction(Settings.EMPTY, "internal:testActionWithNoBlocks", transportService, + clusterService, shardStateAction, threadPool); + listener = new PlainActionFuture<>(); + TestAction.ReroutePhase reroutePhase = testActionWithNoBlocks.new ReroutePhase(task, requestWithTimeout, listener); + reroutePhase.run(); + assertListenerThrows("should fail with an IndexNotFoundException when no blocks", listener, IndexNotFoundException.class); + } + } + + public void testBlocksInPrimaryAction() { + final boolean globalBlock = randomBoolean(); - block = ClusterBlocks.builder().addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, false, - RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); + final TestAction actionWithBlocks = + new TestAction(Settings.EMPTY, "internal:actionWithBlocks", transportService, clusterService, shardStateAction, threadPool) { + @Override + protected ClusterBlockLevel globalBlockLevel() { + return globalBlock ? ClusterBlockLevel.WRITE : null; + } + + @Override + protected ClusterBlockLevel indexBlockLevel() { + return globalBlock == false ? ClusterBlockLevel.WRITE : null; + } + }; + + final String index = "index"; + final ShardId shardId = new ShardId(index, "_na_", 0); + setState(clusterService, stateWithActivePrimary(index, true, randomInt(5))); + + final ClusterBlocks.Builder block = ClusterBlocks.builder(); + if (globalBlock) { + block.addGlobalBlock(new ClusterBlock(randomIntBetween(1, 16), "test global block", randomBoolean(), randomBoolean(), + randomBoolean(), RestStatus.BAD_REQUEST, ClusterBlockLevel.ALL)); + } else { + block.addIndexBlock(index, new ClusterBlock(randomIntBetween(1, 16), "test index block", randomBoolean(), randomBoolean(), + randomBoolean(), RestStatus.FORBIDDEN, ClusterBlockLevel.READ_WRITE)); + } setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block)); - assertListenerThrows("primary phase should fail operation when moving from a retryable block to a non-retryable one", listener, - ClusterBlockException.class); - assertIndexShardUninitialized(); - action = new TestAction(Settings.EMPTY, "internal:testActionWithNoBlocks", transportService, clusterService, shardStateAction, - threadPool) { - @Override - protected ClusterBlockLevel globalBlockLevel() { - return null; - } - }; - listener = new PlainActionFuture<>(); - reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener); - reroutePhase.run(); - assertListenerThrows("should fail with an IndexNotFoundException when no blocks checked", listener, IndexNotFoundException.class); + final ClusterState clusterState = clusterService.state(); + final String targetAllocationID = clusterState.getRoutingTable().shardRoutingTable(shardId).primaryShard().allocationId().getId(); + final long primaryTerm = clusterState.metaData().index(index).primaryTerm(shardId.id()); + final Request request = new Request(shardId); + final ReplicationTask task = maybeTask(); + final PlainActionFuture listener = new PlainActionFuture<>(); + + final TransportReplicationAction.AsyncPrimaryAction asyncPrimaryActionWithBlocks = + actionWithBlocks.new AsyncPrimaryAction(request, targetAllocationID, primaryTerm, createTransportChannel(listener), task); + asyncPrimaryActionWithBlocks.run(); + + final ExecutionException exception = expectThrows(ExecutionException.class, listener::get); + assertThat(exception.getCause(), instanceOf(ClusterBlockException.class)); + assertThat(exception.getCause(), hasToString(containsString("test " + (globalBlock ? "global" : "index") + " block"))); + assertPhase(task, "finished"); } public void assertIndexShardUninitialized() { @@ -377,21 +468,12 @@ public void testClosedIndexOnReroute() throws InterruptedException { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); - ClusterBlockLevel indexBlockLevel = randomBoolean() ? ClusterBlockLevel.WRITE : null; TestAction action = new TestAction(Settings.EMPTY, "internal:testActionWithBlocks", transportService, - clusterService, shardStateAction, threadPool) { - @Override - protected ClusterBlockLevel indexBlockLevel() { - return indexBlockLevel; - } - }; + clusterService, shardStateAction, threadPool); TestAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); - if (indexBlockLevel == ClusterBlockLevel.WRITE) { - assertListenerThrows("must throw block exception", listener, ClusterBlockException.class); - } else { - assertListenerThrows("must throw index closed exception", listener, IndexClosedException.class); - } + assertListenerThrows("must throw index closed exception", listener, IndexClosedException.class); + assertPhase(task, "failed"); assertFalse(request.isRetrySet.get()); } @@ -682,12 +764,12 @@ public void testSeqNoIsSetOnPrimary() throws Exception { PlainActionFuture listener = new PlainActionFuture<>(); - final IndexShard shard = mock(IndexShard.class); + final IndexShard shard = mockIndexShard(shardId, clusterService); when(shard.getPendingPrimaryTerm()).thenReturn(primaryTerm); when(shard.routingEntry()).thenReturn(routingEntry); when(shard.isRelocatedPrimary()).thenReturn(false); IndexShardRoutingTable shardRoutingTable = clusterService.state().routingTable().shardRoutingTable(shardId); - Set inSyncIds = randomBoolean() ? Collections.singleton(routingEntry.allocationId().getId()) : + Set inSyncIds = randomBoolean() ? singleton(routingEntry.allocationId().getId()) : clusterService.state().metaData().index(index).inSyncAllocationIds(0); when(shard.getReplicationGroup()).thenReturn( new ReplicationGroup(shardRoutingTable, @@ -1022,6 +1104,17 @@ protected ReplicaResult shardOperationOnReplica(Request request, IndexShard repl transportService.stop(); } + public void testIsRetryableClusterBlockException() { + final TestAction action = new TestAction(Settings.EMPTY, "internal:testIsRetryableClusterBlockException", transportService, + clusterService, shardStateAction, threadPool); + assertFalse(action.isRetryableClusterBlockException(randomRetryPrimaryException(new ShardId("index", "_na_", 0)))); + + final boolean retryable = randomBoolean(); + ClusterBlock randomBlock = new ClusterBlock(randomIntBetween(1, 16), "test", retryable, randomBoolean(), + randomBoolean(), randomFrom(RestStatus.values()), EnumSet.of(randomFrom(ClusterBlockLevel.values()))); + assertEquals(retryable, action.isRetryableClusterBlockException(new ClusterBlockException(singleton(randomBlock)))); + } + private void assertConcreteShardRequest(TransportRequest capturedRequest, Request expectedRequest, AllocationId expectedAllocationId) { final TransportReplicationAction.ConcreteShardRequest concreteShardRequest = (TransportReplicationAction.ConcreteShardRequest) capturedRequest; @@ -1115,15 +1208,6 @@ private class TestAction extends TransportReplicationAction()), new IndexNameExpressionResolver(), - Request::new, Request::new, ThreadPool.Names.SAME); - } - @Override protected TestResponse newResponseInstance() { return new TestResponse(); @@ -1183,6 +1267,7 @@ final IndexService mockIndexService(final IndexMetaData indexMetaData, ClusterSe private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService) { final IndexShard indexShard = mock(IndexShard.class); + when(indexShard.shardId()).thenReturn(shardId); doAnswer(invocation -> { ActionListener callback = (ActionListener) invocation.getArguments()[0]; count.incrementAndGet(); From 92390c513076658d55813109a152d6e2008137a8 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 22 Nov 2018 15:12:16 +0200 Subject: [PATCH 05/62] Mute test Relates #35822 --- .../client/indexlifecycle/ExplainLifecycleRequestTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java index 2e2aec30e4c66..0cecdbcfa7e9a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java @@ -29,6 +29,7 @@ public class ExplainLifecycleRequestTests extends ESTestCase { + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35822") public void testEqualsAndHashcode() { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copy, this::mutateInstance); } From d4701a428531edc44c13645d928e4b58d69ab254 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Thu, 22 Nov 2018 15:34:53 +0200 Subject: [PATCH 06/62] Mute test InternalEngineTests Relates #35823 --- .../org/elasticsearch/index/engine/InternalEngineTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 25a86f8b211ae..c77a9682fa758 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -67,6 +67,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; @@ -3808,6 +3809,7 @@ public void testSequenceIDs() throws Exception { searchResult.close(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35823") public void testLookupSeqNoByIdInLucene() throws Exception { int numOps = between(10, 100); long seqNo = 0; From ca1b3c6f7d20ca7115b906d266afc9f5ce26c9af Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 22 Nov 2018 14:52:59 +0100 Subject: [PATCH 07/62] [TEST] escape brackets Relates to #35496 --- docs/reference/ccr/apis/get-ccr-stats.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index df4451bfcfbd4..b8491e8a60176 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -105,7 +105,7 @@ The API returns the following results: "number_of_failed_follow_indices" : 0, "number_of_failed_remote_cluster_state_requests" : 0, "number_of_successful_follow_indices" : 1, - "recent_auto_follow_errors" : [ ] + "recent_auto_follow_errors" : [] }, "follow_stats" : { "indices" : [ @@ -150,7 +150,7 @@ The API returns the following results: // TESTRESPONSE[s/"number_of_failed_follow_indices" : 0/"number_of_failed_follow_indices" : $body.auto_follow_stats.number_of_failed_follow_indices/] // TESTRESPONSE[s/"number_of_failed_remote_cluster_state_requests" : 0/"number_of_failed_remote_cluster_state_requests" : $body.auto_follow_stats.number_of_failed_remote_cluster_state_requests/] // TESTRESPONSE[s/"number_of_successful_follow_indices" : 1/"number_of_successful_follow_indices" : $body.auto_follow_stats.number_of_successful_follow_indices/] -// TESTRESPONSE[s/"recent_auto_follow_errors" : [ ]/"recent_auto_follow_errors" : $body.auto_follow_stats.recent_auto_follow_errors/] +// TESTRESPONSE[s/"recent_auto_follow_errors" : \[\]/"recent_auto_follow_errors" : $body.auto_follow_stats.recent_auto_follow_errors/] // TESTRESPONSE[s/"leader_global_checkpoint" : 1024/"leader_global_checkpoint" : $body.follow_stats.indices.0.shards.0.leader_global_checkpoint/] // TESTRESPONSE[s/"leader_max_seq_no" : 1536/"leader_max_seq_no" : $body.follow_stats.indices.0.shards.0.leader_max_seq_no/] // TESTRESPONSE[s/"follower_global_checkpoint" : 768/"follower_global_checkpoint" : $body.follow_stats.indices.0.shards.0.follower_global_checkpoint/] From 121a886c07fb26dfb6f850008d249854f1e6dc15 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 22 Nov 2018 15:42:59 +0100 Subject: [PATCH 08/62] Upgrade to lucene-8.0.0-snapshot-67cdd21996 (#35816) --- buildSrc/version.properties | 2 +- .../ASCIIFoldingTokenFilterFactory.java | 25 +-- .../ArabicNormalizationFilterFactory.java | 8 +- .../BengaliNormalizationFilterFactory.java | 8 +- .../common/CJKWidthFilterFactory.java | 9 +- .../analysis/common/CommonAnalysisPlugin.java | 40 ++--- .../common/DecimalDigitFilterFactory.java | 8 +- .../common/ElisionTokenFilterFactory.java | 8 +- .../GermanNormalizationFilterFactory.java | 8 +- .../HindiNormalizationFilterFactory.java | 8 +- .../IndicNormalizationFilterFactory.java | 8 +- .../common/LowerCaseTokenFilterFactory.java | 8 +- .../common/MappingCharFilterFactory.java | 8 +- .../PatternReplaceCharFilterFactory.java | 14 +- .../PersianNormalizationFilterFactory.java | 8 +- .../ScandinavianFoldingFilterFactory.java | 8 +- ...candinavianNormalizationFilterFactory.java | 8 +- .../SerbianNormalizationFilterFactory.java | 8 +- .../SoraniNormalizationFilterFactory.java | 9 +- .../common/TrimTokenFilterFactory.java | 8 +- .../common/UpperCaseTokenFilterFactory.java | 8 +- .../ASCIIFoldingTokenFilterFactoryTests.java | 5 +- ...essions-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...essions-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...ers-icu-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...ers-icu-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - .../IcuFoldingTokenFilterFactory.java | 6 +- .../IcuNormalizerCharFilterFactory.java | 6 +- .../IcuNormalizerTokenFilterFactory.java | 7 +- .../IcuTransformTokenFilterFactory.java | 6 +- ...uromoji-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...uromoji-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...uromojiIterationMarkCharFilterFactory.java | 6 +- ...rs-nori-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...rs-nori-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...honetic-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...honetic-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...smartcn-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...smartcn-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...stempel-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...stempel-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...fologik-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...fologik-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...-common-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...-common-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...-codecs-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...-codecs-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...ne-core-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...ne-core-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...rouping-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...rouping-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...lighter-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...lighter-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...ne-join-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...ne-join-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...-memory-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...-memory-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...ne-misc-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...ne-misc-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...queries-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...queries-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...yparser-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...yparser-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...sandbox-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...sandbox-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...spatial-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...spatial-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...-extras-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...-extras-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...atial3d-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...atial3d-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - ...suggest-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...suggest-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - .../queries/BinaryDocValuesRangeQuery.java | 2 +- .../apache/lucene/queries/MinDocQuery.java | 2 +- .../queries/SearchAfterSortedDocQuery.java | 2 +- .../analyze/TransportAnalyzeAction.java | 11 +- .../index/analysis/CharFilterFactory.java | 4 + .../index/analysis/CustomAnalyzer.java | 10 +- .../analysis/CustomNormalizerProvider.java | 6 +- ...java => NormalizingCharFilterFactory.java} | 19 ++- .../NormalizingTokenFilterFactory.java | 37 +++++ .../analysis/PreConfiguredCharFilter.java | 9 +- .../analysis/PreConfiguredTokenFilter.java | 14 +- .../analysis/PreConfiguredTokenizer.java | 58 ++----- .../index/analysis/TokenFilterFactory.java | 9 ++ .../index/query/ScriptQueryBuilder.java | 2 +- .../index/shard/ShardSplittingQuery.java | 4 +- .../indices/analysis/AnalysisModule.java | 3 +- .../search/slice/DocValuesSliceQuery.java | 2 +- .../search/slice/TermsSliceQuery.java | 2 +- .../termvectors/GetTermVectorsTests.java | 2 +- .../index/analysis/CustomNormalizerTests.java | 6 +- .../indices/IndicesQueryCacheTests.java | 2 +- .../indices/analysis/AnalysisModuleTests.java | 14 +- .../analysis/AnalysisFactoryTestCase.java | 146 ------------------ ...ne-core-8.0.0-snapshot-67cdd21996.jar.sha1 | 1 + ...ne-core-8.0.0-snapshot-6d9c714052.jar.sha1 | 1 - 98 files changed, 196 insertions(+), 473 deletions(-) create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-analyzers-common-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-analyzers-common-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-grouping-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-join-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-join-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-memory-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-memory-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-misc-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-misc-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-queries-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-queries-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-spatial-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-8.0.0-snapshot-6d9c714052.jar.sha1 create mode 100644 server/licenses/lucene-suggest-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-8.0.0-snapshot-6d9c714052.jar.sha1 rename server/src/main/java/org/elasticsearch/index/analysis/{MultiTermAwareComponent.java => NormalizingCharFilterFactory.java} (69%) create mode 100644 server/src/main/java/org/elasticsearch/index/analysis/NormalizingTokenFilterFactory.java create mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 delete mode 100644 x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 75e95be2a96ee..b65388c2cddb7 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 7.0.0 -lucene = 8.0.0-snapshot-6d9c714052 +lucene = 8.0.0-snapshot-67cdd21996 # optional dependencies spatial4j = 0.7 diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java index 4a5af46feffd2..83e71d5d85897 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java @@ -26,14 +26,14 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; -import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for ASCIIFoldingFilter. */ public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory - implements MultiTermAwareComponent { + implements NormalizingTokenFilterFactory { + public static final ParseField PRESERVE_ORIGINAL = new ParseField("preserve_original"); public static final boolean DEFAULT_PRESERVE_ORIGINAL = false; @@ -51,21 +51,8 @@ public TokenStream create(TokenStream tokenStream) { } @Override - public Object getMultiTermComponent() { - if (preserveOriginal == false) { - return this; - } else { - // See https://issues.apache.org/jira/browse/LUCENE-7536 for the reasoning - return new TokenFilterFactory() { - @Override - public String name() { - return ASCIIFoldingTokenFilterFactory.this.name(); - } - @Override - public TokenStream create(TokenStream tokenStream) { - return new ASCIIFoldingFilter(tokenStream, false); - } - }; - } + public TokenStream normalize(TokenStream tokenStream) { + // Normalization should only emit a single token, so always turn off preserveOriginal + return new ASCIIFoldingFilter(tokenStream, false); } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java index f4e9e2cec347f..08783a89307bd 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ArabicNormalizationFilterFactory.java @@ -24,9 +24,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { ArabicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -37,8 +37,4 @@ public TokenStream create(TokenStream tokenStream) { return new ArabicNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java index fbec142bf3c1b..00b35aa3cdc80 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/BengaliNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link BengaliNormalizationFilter} */ -public class BengaliNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class BengaliNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { BengaliNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new BengaliNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java index 02578a05f8a5b..f7d02c49c6592 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CJKWidthFilterFactory.java @@ -25,9 +25,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); @@ -38,9 +38,4 @@ public TokenStream create(TokenStream tokenStream) { return new CJKWidthFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } - } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 75e0087831a62..a394efdfeb689 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -22,7 +22,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CharArraySet; -import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.ar.ArabicAnalyzer; @@ -492,35 +491,26 @@ public List getPreConfiguredTokenFilters() { @Override public List getPreConfiguredTokenizers() { List tokenizers = new ArrayList<>(); - tokenizers.add(PreConfiguredTokenizer.singleton("keyword", KeywordTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("classic", ClassicTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("uax_url_email", UAX29URLEmailTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("path_hierarchy", PathHierarchyTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("letter", LetterTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("whitespace", WhitespaceTokenizer::new, null)); - tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new, null)); + tokenizers.add(PreConfiguredTokenizer.singleton("keyword", KeywordTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("classic", ClassicTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("uax_url_email", UAX29URLEmailTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("path_hierarchy", PathHierarchyTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("letter", LetterTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("whitespace", WhitespaceTokenizer::new)); + tokenizers.add(PreConfiguredTokenizer.singleton("ngram", NGramTokenizer::new)); tokenizers.add(PreConfiguredTokenizer.singleton("edge_ngram", - () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE), null)); - tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1), null)); - tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new, null)); + () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE))); + tokenizers.add(PreConfiguredTokenizer.singleton("pattern", () -> new PatternTokenizer(Regex.compile("\\W+", null), -1))); + tokenizers.add(PreConfiguredTokenizer.singleton("thai", ThaiTokenizer::new)); // TODO deprecate and remove in API - tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", XLowerCaseTokenizer::new, () -> new TokenFilterFactory() { - @Override - public String name() { - return "lowercase"; - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new LowerCaseFilter(tokenStream); - } - })); + // This is already broken with normalization, so backwards compat isn't necessary? + tokenizers.add(PreConfiguredTokenizer.singleton("lowercase", XLowerCaseTokenizer::new)); // Temporary shim for aliases. TODO deprecate after they are moved - tokenizers.add(PreConfiguredTokenizer.singleton("nGram", NGramTokenizer::new, null)); + tokenizers.add(PreConfiguredTokenizer.singleton("nGram", NGramTokenizer::new)); tokenizers.add(PreConfiguredTokenizer.singleton("edgeNGram", - () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE), null)); - tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new, null)); + () -> new EdgeNGramTokenizer(EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE, EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE))); + tokenizers.add(PreConfiguredTokenizer.singleton("PathHierarchy", PathHierarchyTokenizer::new)); return tokenizers; } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java index 2fa4b91f3c812..9af9f11796cf0 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/DecimalDigitFilterFactory.java @@ -25,12 +25,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link DecimalDigitFilter} */ -public final class DecimalDigitFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public final class DecimalDigitFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { DecimalDigitFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); @@ -41,8 +41,4 @@ public TokenStream create(TokenStream tokenStream) { return new DecimalDigitFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java index d3f920d9e63a2..52cb69952b836 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ElisionTokenFilterFactory.java @@ -27,9 +27,9 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { private final CharArraySet articles; @@ -43,8 +43,4 @@ public TokenStream create(TokenStream tokenStream) { return new ElisionFilter(tokenStream, articles); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java index 1af1e5faaa82f..98748ac2c71b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/GermanNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link GermanNormalizationFilter} */ -public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { GermanNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new GermanNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java index b996d5971e552..e31fa03610540 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HindiNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link HindiNormalizationFilter} */ -public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { HindiNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new HindiNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java index f65c3897e6a8b..155e81717a651 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/IndicNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link IndicNormalizationFilter} */ -public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { IndicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new IndicNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java index f85db0dae685b..7a4f67b55727b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/LowerCaseTokenFilterFactory.java @@ -28,7 +28,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link LowerCaseFilter} and some language-specific variants @@ -39,7 +39,7 @@ *
  • turkish: {@link TurkishLowerCaseFilter} * */ -public class LowerCaseTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class LowerCaseTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { private final String lang; @@ -63,10 +63,6 @@ public TokenStream create(TokenStream tokenStream) { } } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java index 8c8cf93270cf4..c2b361428efe4 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MappingCharFilterFactory.java @@ -26,14 +26,14 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractCharFilterFactory; import org.elasticsearch.index.analysis.Analysis; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingCharFilterFactory; import java.io.Reader; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class MappingCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent { +public class MappingCharFilterFactory extends AbstractCharFilterFactory implements NormalizingCharFilterFactory { private final NormalizeCharMap normMap; @@ -118,8 +118,4 @@ private String parseString(String s) { return new String(out, 0, writePos); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java index 9d3985cc60410..a9708f0beda82 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PatternReplaceCharFilterFactory.java @@ -18,9 +18,6 @@ */ package org.elasticsearch.analysis.common; -import java.io.Reader; -import java.util.regex.Pattern; - import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; @@ -28,9 +25,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractCharFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingCharFilterFactory; + +import java.io.Reader; +import java.util.regex.Pattern; -public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent { +public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory implements NormalizingCharFilterFactory { private final Pattern pattern; private final String replacement; @@ -59,8 +59,4 @@ public Reader create(Reader tokenStream) { return new PatternReplaceCharFilter(pattern, replacement, tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java index 17239b52d97b2..44a20690cee6a 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PersianNormalizationFilterFactory.java @@ -24,9 +24,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { PersianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -37,8 +37,4 @@ public TokenStream create(TokenStream tokenStream) { return new PersianNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java index 6a1dbfdb19228..ce948fa9f8d87 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianFoldingFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link ScandinavianFoldingFilter} */ -public class ScandinavianFoldingFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class ScandinavianFoldingFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { ScandinavianFoldingFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new ScandinavianFoldingFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java index 332dd505e5dca..b5ad4daae81b1 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScandinavianNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link ScandinavianNormalizationFilter} */ -public class ScandinavianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class ScandinavianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { ScandinavianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,8 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new ScandinavianNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java index f6c3a4f55ded5..39c0b065f1fca 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SerbianNormalizationFilterFactory.java @@ -25,9 +25,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { SerbianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -38,8 +38,4 @@ public TokenStream create(TokenStream tokenStream) { return new SerbianNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java index 4cb2fa649fdd3..ba6732aca9b91 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SoraniNormalizationFilterFactory.java @@ -24,12 +24,12 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; /** * Factory for {@link SoraniNormalizationFilter} */ -public class SoraniNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class SoraniNormalizationFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { public SoraniNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -40,9 +40,4 @@ public TokenStream create(TokenStream tokenStream) { return new SoraniNormalizationFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } - } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java index 1412a99f41f44..1ff1f94046812 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TrimTokenFilterFactory.java @@ -25,9 +25,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class TrimTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class TrimTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { private static final String UPDATE_OFFSETS_KEY = "update_offsets"; @@ -43,8 +43,4 @@ public TokenStream create(TokenStream tokenStream) { return new TrimFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java index 7923026d3da7d..2c172d40ff199 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UpperCaseTokenFilterFactory.java @@ -25,9 +25,9 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; -public class UpperCaseTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class UpperCaseTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { public UpperCaseTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); @@ -38,10 +38,6 @@ public TokenStream create(TokenStream tokenStream) { return new UpperCaseFilter(tokenStream); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java index 22ac081011ff4..08d9050f7df95 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.AnalysisTestsHelper; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTokenStreamTestCase; @@ -64,11 +63,9 @@ public void testPreserveOriginal() throws IOException { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); // but the multi-term aware component still emits a single token - tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter) - .getMultiTermComponent(); tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); expected = new String[]{"Anspruche"}; - assertTokenStreamContents(tokenFilter.create(tokenizer), expected); + assertTokenStreamContents(tokenFilter.normalize(tokenizer), expected); } } diff --git a/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-67cdd21996.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..77db008cd5f42 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +65b85d26f4eb4d23b98aaeffc9b1054c23d0227b \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-6d9c714052.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 3c9b97e3e449a..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8f76b85824b273fafa1e25610c3aff66b97b0dd1 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..ba33235486948 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +2c31180c0afaf7ce10244175c68a9189e57b456b \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index ed1ae1538be6b..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee5e4e4341fdde3978b01945bbfaac72a200fa04 \ No newline at end of file diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java index 1997c589bc378..d27ac1d240f6d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java @@ -41,7 +41,7 @@ * * @author kimchy (shay.banon) */ -public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { /** Store here the same Normalizer used by the lucene ICUFoldingFilter */ private static final Normalizer2 ICU_FOLDING_NORMALIZER = Normalizer2.getInstance( ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"), "utr30", Normalizer2.Mode.COMPOSE); @@ -58,8 +58,4 @@ public TokenStream create(TokenStream tokenStream) { return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, normalizer); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java index 86490ff486ecf..f27ff32b6512e 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java @@ -36,7 +36,7 @@ *

    The {@code mode} can be used to provide 'compose' or 'decompose'. Default is compose.

    *

    The {@code unicodeSetFilter} attribute can be used to provide the UniCodeSet for filtering.

    */ -public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent { +public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory implements NormalizingCharFilterFactory { private final Normalizer2 normalizer; @@ -57,8 +57,4 @@ public Reader create(Reader reader) { return new ICUNormalizer2CharFilter(reader, normalizer); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java index 5c23d636dc4a0..66746854c61f4 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java @@ -37,7 +37,7 @@ *

    The {@code name} can be used to provide the type of normalization to perform.

    *

    The {@code unicodeSetFilter} attribute can be used to provide the UniCodeSet for filtering.

    */ -public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(IcuNormalizerTokenFilterFactory.class)); @@ -56,11 +56,6 @@ public TokenStream create(TokenStream tokenStream) { return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, normalizer); } - @Override - public Object getMultiTermComponent() { - return this; - } - static Normalizer2 wrapWithUnicodeSetFilter(final IndexSettings indexSettings, final Normalizer2 normalizer, final Settings settings) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java index 3adced6ffabf3..1ce05be63b0b4 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java @@ -26,7 +26,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { +public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory { private final String id; private final int dir; @@ -45,8 +45,4 @@ public TokenStream create(TokenStream tokenStream) { return new ICUTransformFilter(tokenStream, transliterator); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..ac0faa698b013 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +d39dee7d510aecb9437a1e438ec19cf4398d8792 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 3840dd3256799..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -34dfcdd2e37b62ad01a8bb4fbda66ea6bf513c28 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java index 836dbbdfae219..5d9317e8d5e2b 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java @@ -26,7 +26,7 @@ import java.io.Reader; -public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent { +public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFactory implements NormalizingCharFilterFactory { private final boolean normalizeKanji; private final boolean normalizeKana; @@ -42,8 +42,4 @@ public Reader create(Reader reader) { return new JapaneseIterationMarkCharFilter(reader, normalizeKanji, normalizeKana); } - @Override - public Object getMultiTermComponent() { - return this; - } } diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..149b5d87885b0 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +1f3ce32163fbf344f82d18b61715dc0891c22e00 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 2cd338ed35661..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -25f02c3dfee4efbfe74d87558a6bdd0ea8389e12 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..32ffb9ac968b2 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +6d378fb5b5a904cd3e3a1b1f3bab8b7c5cbc9d85 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 4dd16813c06a3..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1023375e89d6340a93c2409c726a881752eb4ac1 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..3f819cadfd5e9 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +df4957389f85da32b553dd901f30767879a507f2 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index e82b6c54da001..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -70e598154fb5cb3dced5e82de4afcde2009f1755 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..af00cd1da2b1b --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +210ea4e9423e03cd3f6ea9b8e81cab727101d3cb \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 988de653de574..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e8b4634d426efee1515fc289b4ad67d1c714d14d \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-67cdd21996.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..cf2570f59728d --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +3c345959ae03ae458be1590c2ac782b2a621abb2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-6d9c714052.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 67ad39fecd92f..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9f53e03113ca04c337d678126acf025cfeccff6e \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-analyzers-common-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..44e72a2dc11dc --- /dev/null +++ b/server/licenses/lucene-analyzers-common-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +1e557f096cd55fd1f20104b1fb4c0d0095e03fd2 \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-analyzers-common-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 4483e834c284f..0000000000000 --- a/server/licenses/lucene-analyzers-common-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee88dcf4ea69de2a13df7b76d5524e8fd442f243 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-backward-codecs-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..dc35c5c656ebb --- /dev/null +++ b/server/licenses/lucene-backward-codecs-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +77c1844fd0b17e26fb4facb94f6140e98a6bbd49 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-backward-codecs-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index bbdce45b149dc..0000000000000 --- a/server/licenses/lucene-backward-codecs-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec090fd8bd804775aa128ccb20467b062b72d625 \ No newline at end of file diff --git a/server/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..b2ecd56a40a4a --- /dev/null +++ b/server/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +20b559db91bda12f7b242c516915aad26e654baa \ No newline at end of file diff --git a/server/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index cbdbd1acdc715..0000000000000 --- a/server/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0bba71a2e8bfd1c15db407ff06ee4185a091d5ec \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-grouping-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..e364acd45362b --- /dev/null +++ b/server/licenses/lucene-grouping-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +24e4eb6703be36c910bd0d7e3f060259602131b8 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-grouping-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 6c29d732fddea..0000000000000 --- a/server/licenses/lucene-grouping-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fcee5b1586f7c695c65863ca9ee3a8ebe99c3242 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-highlighter-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..6f03dddfef7dd --- /dev/null +++ b/server/licenses/lucene-highlighter-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +1a9acefd0d7a9348f62fb0ea307853fe06cebc63 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-highlighter-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 35cf72f723ede..0000000000000 --- a/server/licenses/lucene-highlighter-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0a26a4870e9fddae497be6899fe9a0a2d3002294 \ No newline at end of file diff --git a/server/licenses/lucene-join-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-join-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..8814180707eb9 --- /dev/null +++ b/server/licenses/lucene-join-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +941fa34281837c5d2a62d67657618b4d6e92c6d7 \ No newline at end of file diff --git a/server/licenses/lucene-join-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-join-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index f2c74d8d8454a..0000000000000 --- a/server/licenses/lucene-join-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -700722c50f8bfcb2d1773b50f43519603961d0ce \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-memory-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..b42afd73bca41 --- /dev/null +++ b/server/licenses/lucene-memory-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +eb78318f2a76b2013857ba72e0ddc42141bad36e \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-memory-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index a3b8ff82ec2bf..0000000000000 --- a/server/licenses/lucene-memory-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9c9657903e4ade7773aaaf76f19d96e2a936e42d \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-misc-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..346ed6def11d8 --- /dev/null +++ b/server/licenses/lucene-misc-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +ce90ede863c08726d7ae70f9f15443f122674d89 \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-misc-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index f7f354945d01c..0000000000000 --- a/server/licenses/lucene-misc-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58ce1753cc41dfe445423c4cee42c129576a2ca2 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-queries-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..3b7b49460f6ff --- /dev/null +++ b/server/licenses/lucene-queries-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +e3b889834b8b43f3c5b718ee0b1b2fd198aa9467 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-queries-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 883c92e08cbea..0000000000000 --- a/server/licenses/lucene-queries-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bf1ee7b66f6e6349624d8760c00669480460a55d \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-queryparser-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..e3f031200f79f --- /dev/null +++ b/server/licenses/lucene-queryparser-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +f4c6c02a0834d582a918c895a715a74f40195297 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-queryparser-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 98a85a1845f75..0000000000000 --- a/server/licenses/lucene-queryparser-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2ed20db0ccc53f966cc211aeb3b623dcf69d2cca \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-sandbox-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..80ed213666b5f --- /dev/null +++ b/server/licenses/lucene-sandbox-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +7ed65e999af74d9356180c91176bcf0bcdf80b6a \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-sandbox-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index a5d5a697df0f3..0000000000000 --- a/server/licenses/lucene-sandbox-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e06d99480f44eede9302fb7dda3c62f3e8ff68e1 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-spatial-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..628aa121e5b47 --- /dev/null +++ b/server/licenses/lucene-spatial-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +28a64cb272639b610064291e726f2a1792c224f2 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-spatial-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index cf1a244c9d336..0000000000000 --- a/server/licenses/lucene-spatial-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64ff3b354c21fc371cfeef208158af92cdf93316 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-spatial-extras-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..c88ae4cc85ec4 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +6af61d6e2d22be8cf0d7afb42ea61e73a59e6708 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-spatial-extras-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 82c8cb3784639..0000000000000 --- a/server/licenses/lucene-spatial-extras-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2dffc0dec40028ca958a0a2fdf0628fd8e8354d0 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-spatial3d-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..d34434a60e7e8 --- /dev/null +++ b/server/licenses/lucene-spatial3d-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +7e7d3d4c5b7a3a4a065db5c7e4a22d75c11191ff \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-spatial3d-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index a23c263e7a337..0000000000000 --- a/server/licenses/lucene-spatial3d-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d0ed3d77875bab18abe45706ec8b5d441cf46bdc \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.0.0-snapshot-67cdd21996.jar.sha1 b/server/licenses/lucene-suggest-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..2464488f65e5d --- /dev/null +++ b/server/licenses/lucene-suggest-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +b2993443ae730960c22a2c9050f58d943fb8797c \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.0.0-snapshot-6d9c714052.jar.sha1 b/server/licenses/lucene-suggest-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index 7d48c041d64ce..0000000000000 --- a/server/licenses/lucene-suggest-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bb05a98bb9c2615ad1262980dd6b07802bafa1d \ No newline at end of file diff --git a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java index 63db15b2ee168..9fa9bcacc0ac6 100644 --- a/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/BinaryDocValuesRangeQuery.java @@ -108,7 +108,7 @@ public float matchCost() { return 4; // at most 4 comparisons } }; - return new ConstantScoreScorer(this, score(), iterator); + return new ConstantScoreScorer(this, score(), scoreMode, iterator); } @Override diff --git a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java index b9a001b6e7370..dac69eaca23a0 100644 --- a/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/MinDocQuery.java @@ -92,7 +92,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } final int segmentMinDoc = Math.max(0, minDoc - context.docBase); final DocIdSetIterator disi = new MinDocIterator(segmentMinDoc, maxDoc); - return new ConstantScoreScorer(this, score(), disi); + return new ConstantScoreScorer(this, score(), scoreMode, disi); } @Override diff --git a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java b/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java index 2c436f0227222..6497739c48cac 100644 --- a/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java +++ b/server/src/main/java/org/apache/lucene/queries/SearchAfterSortedDocQuery.java @@ -87,7 +87,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return null; } final DocIdSetIterator disi = new MinDocQuery.MinDocIterator(firstDoc, maxDoc); - return new ConstantScoreScorer(this, score(), disi); + return new ConstantScoreScorer(this, score(), scoreMode, disi); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index d360bc45b8763..9538bd4b4d22c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -27,7 +27,6 @@ import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.support.ActionFilters; @@ -42,6 +41,7 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; @@ -49,7 +49,8 @@ import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.CustomAnalyzer; import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.NormalizingCharFilterFactory; +import org.elasticsearch.index.analysis.NormalizingTokenFilterFactory; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.analysis.TokenizerFactory; @@ -575,11 +576,10 @@ private static List parseCharFilterFactories(AnalyzeRequest r throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]"); } if (normalizer) { - if (charFilterFactory instanceof MultiTermAwareComponent == false) { + if (charFilterFactory instanceof NormalizingCharFilterFactory == false) { throw new IllegalArgumentException("Custom normalizer may not use char filter [" + charFilterFactory.name() + "]"); } - charFilterFactory = (CharFilterFactory) ((MultiTermAwareComponent) charFilterFactory).getMultiTermComponent(); } charFilterFactoryList.add(charFilterFactory); } @@ -677,11 +677,10 @@ private static List parseTokenFilterFactories(AnalyzeRequest throw new IllegalArgumentException("failed to find or create token filter under [" + tokenFilter.name + "]"); } if (normalizer) { - if (tokenFilterFactory instanceof MultiTermAwareComponent == false) { + if (tokenFilterFactory instanceof NormalizingTokenFilterFactory == false) { throw new IllegalArgumentException("Custom normalizer may not use filter [" + tokenFilterFactory.name() + "]"); } - tokenFilterFactory = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilterFactory).getMultiTermComponent(); } tokenFilterFactoryList.add(tokenFilterFactory); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CharFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/CharFilterFactory.java index 6f85615c95e1b..fd90402ab18cd 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CharFilterFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CharFilterFactory.java @@ -26,4 +26,8 @@ public interface CharFilterFactory { String name(); Reader create(Reader reader); + + default Reader normalize(Reader reader) { + return reader; + } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java index d70b4628f532c..87a540312b411 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java @@ -107,10 +107,7 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected Reader initReaderForNormalization(String fieldName, Reader reader) { for (CharFilterFactory charFilter : charFilters) { - if (charFilter instanceof MultiTermAwareComponent) { - charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent(); - reader = charFilter.create(reader); - } + reader = charFilter.normalize(reader); } return reader; } @@ -119,10 +116,7 @@ protected Reader initReaderForNormalization(String fieldName, Reader reader) { protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = in; for (TokenFilterFactory filter : tokenFilters) { - if (filter instanceof MultiTermAwareComponent) { - filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent(); - result = filter.create(result); - } + result = filter.normalize(result); } return result; } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java b/server/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java index 13946be3a8d22..13ee76e8d6265 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java @@ -57,11 +57,10 @@ public void build(final String tokenizerName, final TokenizerFactory tokenizerFa throw new IllegalArgumentException("Custom normalizer [" + name() + "] failed to find char_filter under name [" + charFilterName + "]"); } - if (charFilter instanceof MultiTermAwareComponent == false) { + if (charFilter instanceof NormalizingCharFilterFactory == false) { throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use char filter [" + charFilterName + "]"); } - charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent(); charFiltersList.add(charFilter); } @@ -73,10 +72,9 @@ public void build(final String tokenizerName, final TokenizerFactory tokenizerFa throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name [" + tokenFilterName + "]"); } - if (tokenFilter instanceof MultiTermAwareComponent == false) { + if (tokenFilter instanceof NormalizingTokenFilterFactory == false) { throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use filter [" + tokenFilterName + "]"); } - tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter).getMultiTermComponent(); tokenFilterList.add(tokenFilter); } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java b/server/src/main/java/org/elasticsearch/index/analysis/NormalizingCharFilterFactory.java similarity index 69% rename from server/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java rename to server/src/main/java/org/elasticsearch/index/analysis/NormalizingCharFilterFactory.java index ca1d4b9e820aa..145c5a8b416d5 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/NormalizingCharFilterFactory.java @@ -19,12 +19,19 @@ package org.elasticsearch.index.analysis; -/** Elasticsearch counterpart of {@link org.apache.lucene.analysis.util.MultiTermAwareComponent}. */ -public interface MultiTermAwareComponent { +import java.io.Reader; - /** Returns an analysis component to handle analysis if multi-term queries. - * The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory. - */ - Object getMultiTermComponent(); +/** + * A CharFilterFactory that also supports normalization + * + * The default implementation of {@link #normalize(Reader)} delegates to + * {@link #create(Reader)} + */ +public interface NormalizingCharFilterFactory extends CharFilterFactory { + + @Override + default Reader normalize(Reader reader) { + return create(reader); + } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/NormalizingTokenFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/NormalizingTokenFilterFactory.java new file mode 100644 index 0000000000000..0e8dbdf433afb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/analysis/NormalizingTokenFilterFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.TokenStream; + +/** + * A TokenFilterFactory that may be used for normalization + * + * The default implementation delegates {@link #normalize(TokenStream)} to + * {@link #create(TokenStream)}}. + */ +public interface NormalizingTokenFilterFactory extends TokenFilterFactory { + + @Override + default TokenStream normalize(TokenStream tokenStream) { + return create(tokenStream); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredCharFilter.java b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredCharFilter.java index a3fddce3e060f..7487b1d8e8d2b 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredCharFilter.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredCharFilter.java @@ -83,12 +83,10 @@ public boolean shouldUseFilterForMultitermQueries() { return useFilterForMultitermQueries; } - private interface MultiTermAwareCharFilterFactory extends CharFilterFactory, MultiTermAwareComponent {} - @Override protected CharFilterFactory create(Version version) { if (useFilterForMultitermQueries) { - return new MultiTermAwareCharFilterFactory() { + return new NormalizingCharFilterFactory() { @Override public String name() { return getName(); @@ -98,11 +96,6 @@ public String name() { public Reader create(Reader reader) { return create.apply(reader, version); } - - @Override - public Object getMultiTermComponent() { - return this; - } }; } return new CharFilterFactory() { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java index 12130e856f32a..bd70d9295556d 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java @@ -84,12 +84,16 @@ public boolean shouldUseFilterForMultitermQueries() { return useFilterForMultitermQueries; } - private interface MultiTermAwareTokenFilterFactory extends TokenFilterFactory, MultiTermAwareComponent {} - @Override protected TokenFilterFactory create(Version version) { if (useFilterForMultitermQueries) { - return new MultiTermAwareTokenFilterFactory() { + return new NormalizingTokenFilterFactory() { + + @Override + public TokenStream normalize(TokenStream tokenStream) { + return create.apply(tokenStream, version); + } + @Override public String name() { return getName(); @@ -100,10 +104,6 @@ public TokenStream create(TokenStream tokenStream) { return create.apply(tokenStream, version); } - @Override - public Object getMultiTermComponent() { - return this; - } }; } return new TokenFilterFactory() { diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenizer.java b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenizer.java index 131246d0b766a..6dadf9c117182 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenizer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenizer.java @@ -21,7 +21,6 @@ import org.apache.lucene.analysis.Tokenizer; import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory; import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; @@ -37,77 +36,40 @@ public final class PreConfiguredTokenizer extends PreConfiguredAnalysisComponent * * @param name the name of the tokenizer in the api * @param create builds the tokenizer - * @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the - * {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries. */ - public static PreConfiguredTokenizer singleton(String name, Supplier create, - @Nullable Supplier multiTermComponent) { - return new PreConfiguredTokenizer(name, CachingStrategy.ONE, version -> create.get(), - multiTermComponent == null ? null : version -> multiTermComponent.get()); + public static PreConfiguredTokenizer singleton(String name, Supplier create) { + return new PreConfiguredTokenizer(name, CachingStrategy.ONE, version -> create.get()); } /** * Create a pre-configured tokenizer that may vary based on the Lucene version. - * + * * @param name the name of the tokenizer in the api * @param create builds the tokenizer - * @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the - * {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries. */ - public static PreConfiguredTokenizer luceneVersion(String name, Function create, - @Nullable Function multiTermComponent) { - return new PreConfiguredTokenizer(name, CachingStrategy.LUCENE, version -> create.apply(version.luceneVersion), - multiTermComponent == null ? null : version -> multiTermComponent.apply(version.luceneVersion)); + public static PreConfiguredTokenizer luceneVersion(String name, Function create) { + return new PreConfiguredTokenizer(name, CachingStrategy.LUCENE, version -> create.apply(version.luceneVersion)); } /** * Create a pre-configured tokenizer that may vary based on the Elasticsearch version. - * + * * @param name the name of the tokenizer in the api * @param create builds the tokenizer - * @param multiTermComponent null if this tokenizer shouldn't be used for multi-term queries, otherwise a supplier for the - * {@link TokenFilterFactory} that stands in for this tokenizer in multi-term queries. */ - public static PreConfiguredTokenizer elasticsearchVersion(String name, Function create, - @Nullable Function multiTermComponent) { - return new PreConfiguredTokenizer(name, CachingStrategy.ELASTICSEARCH, create, multiTermComponent); + public static PreConfiguredTokenizer elasticsearchVersion(String name, Function create) { + return new PreConfiguredTokenizer(name, CachingStrategy.ELASTICSEARCH, create); } private final Function create; - private final Function multiTermComponent; - private PreConfiguredTokenizer(String name, PreBuiltCacheFactory.CachingStrategy cache, Function create, - @Nullable Function multiTermComponent) { + private PreConfiguredTokenizer(String name, PreBuiltCacheFactory.CachingStrategy cache, Function create) { super(name, cache); this.create = create; - this.multiTermComponent = multiTermComponent; - } - - /** - * Does this tokenizer has an equivalent component for analyzing multi-term queries? - */ - public boolean hasMultiTermComponent() { - return multiTermComponent != null; } - private interface MultiTermAwareTokenizerFactory extends TokenizerFactory, MultiTermAwareComponent {} - @Override protected TokenizerFactory create(Version version) { - if (multiTermComponent != null) { - return new MultiTermAwareTokenizerFactory() { - @Override - public Tokenizer create() { - return create.apply(version); - } - - @Override - public Object getMultiTermComponent() { - return multiTermComponent.apply(version); - } - }; - } else { - return () -> create.apply(version); - } + return () -> create.apply(version); } } diff --git a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index 9d9a48c3a332e..a400755c860a1 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -31,6 +31,15 @@ public interface TokenFilterFactory { TokenStream create(TokenStream tokenStream); + /** + * Normalize a tokenStream for use in multi-term queries + * + * The default implementation is a no-op + */ + default TokenStream normalize(TokenStream tokenStream) { + return tokenStream; + } + /** * Does this analyzer mess up the {@link OffsetAttribute}s in such as way as to break the * {@link FastVectorHighlighter}? If this is {@code true} then the diff --git a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java index 50586aa2522ad..8a3666afb9d12 100644 --- a/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java @@ -191,7 +191,7 @@ public float matchCost() { return 1000f; } }; - return new ConstantScoreScorer(this, score(), twoPhase); + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index a22193974272c..fb33cceaa49d8 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -114,7 +114,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { TwoPhaseIterator twoPhaseIterator = parentBitSet == null ? new RoutingPartitionedDocIdSetIterator(visitor) : new NestedRoutingPartitionedDocIdSetIterator(visitor, parentBitSet); - return new ConstantScoreScorer(this, score(), twoPhaseIterator); + return new ConstantScoreScorer(this, score(), scoreMode, twoPhaseIterator); } else { // here we potentially guard the docID consumers with our parent bitset if we have one. // this ensures that we are only marking root documents in the nested case and if necessary @@ -155,7 +155,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } } - return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length())); + return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bitSet, bitSet.length())); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index 2bfde42a3e876..60a2b1640ed5b 100644 --- a/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/server/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -212,8 +212,7 @@ static Map setupPreConfiguredTokenizers(List tokenizer.create(Version.CURRENT), null); + preConfigured = PreConfiguredTokenizer.singleton(name, () -> tokenizer.create(Version.CURRENT)); break; default: throw new UnsupportedOperationException( diff --git a/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java index f2cf854947fd8..86d778625a260 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java +++ b/server/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java @@ -75,7 +75,7 @@ public float matchCost() { return 10; } }; - return new ConstantScoreScorer(this, score(), twoPhase); + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java index 1a10770fe9d2b..52ac72d01b778 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java +++ b/server/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java @@ -62,7 +62,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo public Scorer scorer(LeafReaderContext context) throws IOException { final DocIdSet disi = build(context.reader()); final DocIdSetIterator leafIt = disi.iterator(); - return new ConstantScoreScorer(this, score(), leafIt); + return new ConstantScoreScorer(this, score(), scoreMode, leafIt); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java index 0e8877701e4b9..6379867d51604 100644 --- a/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java +++ b/server/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java @@ -98,7 +98,7 @@ public TokenStream create(TokenStream tokenStream) { @Override public List getPreConfiguredTokenizers() { return Collections.singletonList(PreConfiguredTokenizer.singleton("mock-whitespace", - () -> new MockTokenizer(MockTokenizer.WHITESPACE, false), null)); + () -> new MockTokenizer(MockTokenizer.WHITESPACE, false))); } // Based on DelimitedPayloadTokenFilter: diff --git a/server/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java index 1dcb2d4e39fd6..697c39bb90d30 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java @@ -137,7 +137,7 @@ public List getPreConfiguredCharFilters() { @Override public Map> getCharFilters() { return singletonMap("mock_char_filter", (indexSettings, env, name, settings) -> { - class Factory implements CharFilterFactory, MultiTermAwareComponent { + class Factory implements NormalizingCharFilterFactory { @Override public String name() { return name; @@ -162,10 +162,6 @@ public void close() throws IOException { } }; } - @Override - public Object getMultiTermComponent() { - return this; - } } return new Factory(); }); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java index 88bc4381626d4..5185eaebe4306 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java @@ -78,7 +78,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo return new ConstantScoreWeight(this, boost) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc())); + return new ConstantScoreScorer(this, score(), scoreMode, DocIdSetIterator.all(context.reader().maxDoc())); } @Override diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java index f3a8847a9b594..c769da0af4d1e 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java @@ -371,9 +371,6 @@ public List getPreConfiguredTokenFilters() { * and that do not vary based on version at all. */ public void testPluginPreConfiguredTokenizers() throws IOException { - boolean noVersionSupportsMultiTerm = randomBoolean(); - boolean luceneVersionSupportsMultiTerm = randomBoolean(); - boolean elasticsearchVersionSupportsMultiTerm = randomBoolean(); // Simple tokenizer that always spits out a single token with some preconfigured characters final class FixedTokenizer extends Tokenizer { @@ -409,16 +406,11 @@ public void reset() throws IOException { @Override public List getPreConfiguredTokenizers() { return Arrays.asList( - PreConfiguredTokenizer.singleton("no_version", () -> new FixedTokenizer("no_version"), - noVersionSupportsMultiTerm ? () -> AppendTokenFilter.factoryForSuffix("no_version") : null), + PreConfiguredTokenizer.singleton("no_version", () -> new FixedTokenizer("no_version")), PreConfiguredTokenizer.luceneVersion("lucene_version", - luceneVersion -> new FixedTokenizer(luceneVersion.toString()), - luceneVersionSupportsMultiTerm ? - luceneVersion -> AppendTokenFilter.factoryForSuffix(luceneVersion.toString()) : null), + luceneVersion -> new FixedTokenizer(luceneVersion.toString())), PreConfiguredTokenizer.elasticsearchVersion("elasticsearch_version", - esVersion -> new FixedTokenizer(esVersion.toString()), - elasticsearchVersionSupportsMultiTerm ? - esVersion -> AppendTokenFilter.factoryForSuffix(esVersion.toString()) : null) + esVersion -> new FixedTokenizer(esVersion.toString())) ); } })).getAnalysisRegistry(); diff --git a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java index 63ec090dcc65b..0238526cab4dd 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/analysis/AnalysisFactoryTestCase.java @@ -19,24 +19,17 @@ package org.elasticsearch.indices.analysis; -import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.index.analysis.HunspellTokenFilterFactory; -import org.elasticsearch.index.analysis.MultiTermAwareComponent; -import org.elasticsearch.index.analysis.PreConfiguredCharFilter; -import org.elasticsearch.index.analysis.PreConfiguredTokenFilter; -import org.elasticsearch.index.analysis.PreConfiguredTokenizer; import org.elasticsearch.index.analysis.ShingleTokenFilterFactory; import org.elasticsearch.index.analysis.StandardTokenizerFactory; import org.elasticsearch.index.analysis.StopTokenFilterFactory; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -46,9 +39,6 @@ import java.util.regex.Pattern; import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.typeCompatibleWith; /** * Alerts us if new analysis components are added to Lucene, so we don't miss them. @@ -308,142 +298,6 @@ public void testTokenFilters() { assertTrue("new tokenfilters found, please update KNOWN_TOKENFILTERS: " + missing.toString(), missing.isEmpty()); } - public void testMultiTermAware() { - Collection> expected = new HashSet<>(); - for (Map.Entry> entry : getTokenizers().entrySet()) { - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom( - org.apache.lucene.analysis.util.TokenizerFactory.lookupClass(entry.getKey()))) { - expected.add(entry.getValue()); - } - } - for (Map.Entry> entry : getTokenFilters().entrySet()) { - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom( - org.apache.lucene.analysis.util.TokenFilterFactory.lookupClass(entry.getKey()))) { - expected.add(entry.getValue()); - } - } - for (Map.Entry> entry : getCharFilters().entrySet()) { - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom( - org.apache.lucene.analysis.util.CharFilterFactory.lookupClass(entry.getKey()))) { - expected.add(entry.getValue()); - } - } - expected.remove(Void.class); - expected.remove(MovedToAnalysisCommon.class); - expected.remove(Deprecated.class); - - Collection> actual = new HashSet<>(); - for (Class clazz : getTokenizers().values()) { - if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) { - actual.add(clazz); - } - } - for (Class clazz : getTokenFilters().values()) { - if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) { - actual.add(clazz); - } - } - for (Class clazz : getCharFilters().values()) { - if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) { - actual.add(clazz); - } - } - - Set> classesMissingMultiTermSupport = new HashSet<>(expected); - classesMissingMultiTermSupport.removeAll(actual); - assertTrue("Classes are missing multi-term support: " + classesMissingMultiTermSupport, - classesMissingMultiTermSupport.isEmpty()); - - Set> classesThatShouldNotHaveMultiTermSupport = new HashSet<>(actual); - classesThatShouldNotHaveMultiTermSupport.removeAll(expected); - assertTrue("Classes should not have multi-term support: " + classesThatShouldNotHaveMultiTermSupport, - classesThatShouldNotHaveMultiTermSupport.isEmpty()); - } - - public void testPreBuiltMultiTermAware() { - Collection expected = new HashSet<>(); - Collection actual = new HashSet<>(); - - Map preConfiguredTokenFilters = - new HashMap<>(AnalysisModule.setupPreConfiguredTokenFilters(singletonList(plugin))); - for (Map.Entry> entry : getPreConfiguredTokenFilters().entrySet()) { - String name = entry.getKey(); - Class luceneFactory = entry.getValue(); - PreConfiguredTokenFilter filter = preConfiguredTokenFilters.remove(name); - assertNotNull("test claims pre built token filter [" + name + "] should be available but it wasn't", filter); - if (luceneFactory == Void.class) { - continue; - } - if (luceneFactory == null) { - luceneFactory = TokenFilterFactory.lookupClass(toCamelCase(name)); - } - assertThat(luceneFactory, typeCompatibleWith(TokenFilterFactory.class)); - if (filter.shouldUseFilterForMultitermQueries()) { - actual.add("token filter [" + name + "]"); - } - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) { - expected.add("token filter [" + name + "]"); - } - } - assertThat("pre configured token filter not registered with test", preConfiguredTokenFilters.keySet(), empty()); - - Map preConfiguredTokenizers = new HashMap<>( - AnalysisModule.setupPreConfiguredTokenizers(singletonList(plugin))); - for (Map.Entry> entry : getPreConfiguredTokenizers().entrySet()) { - String name = entry.getKey(); - Class luceneFactory = entry.getValue(); - PreConfiguredTokenizer tokenizer = preConfiguredTokenizers.remove(name); - assertNotNull("test claims pre built tokenizer [" + name + "] should be available but it wasn't", tokenizer); - if (luceneFactory == Void.class) { - continue; - } - if (luceneFactory == null) { - luceneFactory = TokenizerFactory.lookupClass(toCamelCase(name)); - } - assertThat(luceneFactory, typeCompatibleWith(TokenizerFactory.class)); - if (tokenizer.hasMultiTermComponent()) { - actual.add(tokenizer); - } - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) { - expected.add(tokenizer); - } - } - assertThat("pre configured tokenizer not registered with test", preConfiguredTokenizers.keySet(), empty()); - - Map preConfiguredCharFilters = new HashMap<>( - AnalysisModule.setupPreConfiguredCharFilters(singletonList(plugin))); - for (Map.Entry> entry : getPreConfiguredCharFilters().entrySet()) { - String name = entry.getKey(); - Class luceneFactory = entry.getValue(); - PreConfiguredCharFilter filter = preConfiguredCharFilters.remove(name); - assertNotNull("test claims pre built char filter [" + name + "] should be available but it wasn't", filter); - if (luceneFactory == Void.class) { - continue; - } - if (luceneFactory == null) { - luceneFactory = TokenFilterFactory.lookupClass(toCamelCase(name)); - } - assertThat(luceneFactory, typeCompatibleWith(CharFilterFactory.class)); - if (filter.shouldUseFilterForMultitermQueries()) { - actual.add(filter); - } - if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(luceneFactory)) { - expected.add("token filter [" + name + "]"); - } - } - assertThat("pre configured char filter not registered with test", preConfiguredCharFilters.keySet(), empty()); - - Set classesMissingMultiTermSupport = new HashSet<>(expected); - classesMissingMultiTermSupport.removeAll(actual); - assertTrue("Pre-built components are missing multi-term support: " + classesMissingMultiTermSupport, - classesMissingMultiTermSupport.isEmpty()); - - Set classesThatShouldNotHaveMultiTermSupport = new HashSet<>(actual); - classesThatShouldNotHaveMultiTermSupport.removeAll(expected); - assertTrue("Pre-built components should not have multi-term support: " + classesThatShouldNotHaveMultiTermSupport, - classesThatShouldNotHaveMultiTermSupport.isEmpty()); - } - /** * Marker class for components that have moved to the analysis-common modules. This will be * removed when the module is complete and these analysis components aren't available to core. diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 new file mode 100644 index 0000000000000..b2ecd56a40a4a --- /dev/null +++ b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-67cdd21996.jar.sha1 @@ -0,0 +1 @@ +20b559db91bda12f7b242c516915aad26e654baa \ No newline at end of file diff --git a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 b/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 deleted file mode 100644 index cbdbd1acdc715..0000000000000 --- a/x-pack/plugin/sql/sql-action/licenses/lucene-core-8.0.0-snapshot-6d9c714052.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0bba71a2e8bfd1c15db407ff06ee4185a091d5ec \ No newline at end of file From f8a7bf6339f80ff949143f2c091339a1766e4c1a Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 22 Nov 2018 15:08:46 +0000 Subject: [PATCH 09/62] Remove unnecessary throws IOException in CompressedXContent.string() (#35821) --- .../common/compress/CompressedXContent.java | 8 ++------ .../elasticsearch/index/mapper/MapperService.java | 8 +------- .../org/elasticsearch/indices/IndicesService.java | 14 +++++--------- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java index 991dfccf7f994..9bc3eab00a564 100644 --- a/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java +++ b/server/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java @@ -146,7 +146,7 @@ public byte[] uncompressed() { } } - public String string() throws IOException { + public String string() { return new BytesRef(uncompressed()).utf8ToString(); } @@ -188,10 +188,6 @@ public int hashCode() { @Override public String toString() { - try { - return string(); - } catch (IOException e) { - return "_na_"; - } + return string(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 076f6e7ebe030..6aab34c5f7676 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -21,14 +21,12 @@ import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.Term; import org.elasticsearch.Assertions; -import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -336,11 +334,7 @@ private synchronized Map internalMerge(Map { assert recoveryState.getRecoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS: "mapping update consumer only required by local shards recovery"; - try { - client.admin().indices().preparePutMapping() - .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid - .setType(type) - .setSource(mapping.source().string(), XContentType.JSON) - .get(); - } catch (IOException ex) { - throw new ElasticsearchException("failed to stringify mapping source", ex); - } + client.admin().indices().preparePutMapping() + .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid + .setType(type) + .setSource(mapping.source().string(), XContentType.JSON) + .get(); }, this); return indexShard; } From 9870c7491c8ac0b50cc39266f370141eca81f4a5 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 22 Nov 2018 16:23:31 +0000 Subject: [PATCH 10/62] [ML] Add docs for ML info endpoint (#35783) This endpoint was not previously documented as it was not particularly useful to end users. However, since the HLRC will support the endpoint we need some documentation to link to. The purpose of the endpoint is to provide defaults and limits used by ML. These are needed to fully understand configurations that have missing values because the missing value means the default should be used. Relates #35777 --- docs/reference/ml/apis/get-ml-info.asciidoc | 60 +++++++++++++++++++++ docs/reference/ml/apis/ml-api.asciidoc | 7 +++ 2 files changed, 67 insertions(+) create mode 100644 docs/reference/ml/apis/get-ml-info.asciidoc diff --git a/docs/reference/ml/apis/get-ml-info.asciidoc b/docs/reference/ml/apis/get-ml-info.asciidoc new file mode 100644 index 0000000000000..d3f9e69560bc9 --- /dev/null +++ b/docs/reference/ml/apis/get-ml-info.asciidoc @@ -0,0 +1,60 @@ +[role="xpack"] +[testenv="platinum"] +[[get-ml-info]] +=== Get Machine Learning Info API +++++ +Get Machine Learning Info +++++ + +Returns defaults and limits used by machine learning. + +==== Request + +`GET _xpack/ml/info` + +==== Description + +This endpoint is designed to be used by a user interface that needs to fully +understand machine learning configurations where some options are not specified, +meaning that the defaults should be used. This endpoint may be used to find out +what those defaults are. + + +==== Authorization + +You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster +privileges to use this API. The `machine_learning_admin` and `machine_learning_user` +roles provide these privileges. For more information, see +{stack-ov}/security-privileges.html[Security Privileges] and +{stack-ov}/built-in-roles.html[Built-in Roles]. + + +==== Examples + +The endpoint takes no arguments: + +[source,js] +-------------------------------------------------- +GET _xpack/ml/info +-------------------------------------------------- +// CONSOLE +// TEST + +This is a possible response: +[source,js] +---- +{ + "defaults" : { + "anomaly_detectors" : { + "model_memory_limit" : "1gb", + "categorization_examples_limit" : 4, + "model_snapshot_retention_days" : 1 + }, + "datafeeds" : { + "scroll_size" : 1000 + } + }, + "limits" : { } +} +---- +// TESTRESPONSE diff --git a/docs/reference/ml/apis/ml-api.asciidoc b/docs/reference/ml/apis/ml-api.asciidoc index bb086435fb24c..d3d1c42d0a8e3 100644 --- a/docs/reference/ml/apis/ml-api.asciidoc +++ b/docs/reference/ml/apis/ml-api.asciidoc @@ -76,6 +76,12 @@ machine learning APIs and in advanced job configuration options in Kibana. * <> +[float] +[[ml-api-ml-info-endpoint]] +=== Info + +* <> + //ADD include::post-calendar-event.asciidoc[] include::put-calendar-job.asciidoc[] @@ -111,6 +117,7 @@ include::get-datafeed-stats.asciidoc[] include::get-influencer.asciidoc[] include::get-job.asciidoc[] include::get-job-stats.asciidoc[] +include::get-ml-info.asciidoc[] include::get-snapshot.asciidoc[] include::get-calendar-event.asciidoc[] include::get-filter.asciidoc[] From b9cba859d12aa95d44334727943dbee9a8b995bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 23 Nov 2018 05:03:19 +0100 Subject: [PATCH 11/62] [Tests] Fix creating ExplainLifecycleRequest with no indices (#35828) We didn't check that the ExplainLifecycleRequest was constructed with at least one index before, now that we do we must also make sure the tests mutateInstance() method used in equals/hashCode checks doesn't accidentally create an empty index array. Closes #35822 --- .../client/indexlifecycle/ExplainLifecycleRequestTests.java | 3 +-- .../core/indexlifecycle/ExplainLifecycleRequestTests.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java index 0cecdbcfa7e9a..6148bd91a3734 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java @@ -29,7 +29,6 @@ public class ExplainLifecycleRequestTests extends ESTestCase { - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35822") public void testEqualsAndHashcode() { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), this::copy, this::mutateInstance); } @@ -55,7 +54,7 @@ private ExplainLifecycleRequest mutateInstance(ExplainLifecycleRequest instance) switch (between(0, 1)) { case 0: indices = randomValueOtherThanMany(i -> Arrays.equals(i, instance.getIndices()), - () -> generateRandomStringArray(20, 10, false, true)); + () -> generateRandomStringArray(20, 10, false, false)); break; case 1: indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java index 490bfbf0bf380..4c1ffac49a244 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ExplainLifecycleRequestTests.java @@ -19,7 +19,7 @@ public class ExplainLifecycleRequestTests extends AbstractWireSerializingTestCas protected ExplainLifecycleRequest createTestInstance() { ExplainLifecycleRequest request = new ExplainLifecycleRequest(); if (randomBoolean()) { - request.indices(generateRandomStringArray(20, 20, false, true)); + request.indices(generateRandomStringArray(20, 20, false, false)); } if (randomBoolean()) { IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), From 51351c5468c33ec513a0e4ae35a128083db5f132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Campinas?= Date: Fri, 23 Nov 2018 05:29:20 +0100 Subject: [PATCH 12/62] [Docs] Correct template example description #35829 --- docs/java-api/search.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/java-api/search.asciidoc b/docs/java-api/search.asciidoc index f24e479a79313..73995028db427 100644 --- a/docs/java-api/search.asciidoc +++ b/docs/java-api/search.asciidoc @@ -242,7 +242,7 @@ sr = new SearchTemplateRequestBuilder(client) .get() <5> .getResponse(); <6> -------------------------------------------------- -<1> template name +<1> template's body <2> template is passed inline <3> parameters <4> set the execution context (ie. define the index name here) From c17fa7fd6134eb27bd77014cb843341672b72c35 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 23 Nov 2018 07:52:29 +0100 Subject: [PATCH 13/62] Fixed response classes in hlrc docs --- docs/java-rest/high-level/ccr/pause_follow.asciidoc | 2 +- docs/java-rest/high-level/ccr/resume_follow.asciidoc | 2 +- docs/java-rest/high-level/ccr/unfollow.asciidoc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/java-rest/high-level/ccr/pause_follow.asciidoc b/docs/java-rest/high-level/ccr/pause_follow.asciidoc index 08acf7cadce8a..de81afa1e83b3 100644 --- a/docs/java-rest/high-level/ccr/pause_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/pause_follow.asciidoc @@ -1,7 +1,7 @@ -- :api: ccr-pause-follow :request: PauseFollowRequest -:response: PauseFollowResponse +:response: AcknowledgedResponse -- [id="{upid}-{api}"] diff --git a/docs/java-rest/high-level/ccr/resume_follow.asciidoc b/docs/java-rest/high-level/ccr/resume_follow.asciidoc index 349440dbc9450..18d69b69d4979 100644 --- a/docs/java-rest/high-level/ccr/resume_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/resume_follow.asciidoc @@ -1,7 +1,7 @@ -- :api: ccr-resume-follow :request: ResumeFollowRequest -:response: ResumeFollowResponse +:response: AcknowledgedResponse -- [id="{upid}-{api}"] diff --git a/docs/java-rest/high-level/ccr/unfollow.asciidoc b/docs/java-rest/high-level/ccr/unfollow.asciidoc index bb6dd654ed4d1..779b8c3f586c4 100644 --- a/docs/java-rest/high-level/ccr/unfollow.asciidoc +++ b/docs/java-rest/high-level/ccr/unfollow.asciidoc @@ -1,7 +1,7 @@ -- :api: ccr-unfollow :request: UnfollowRequest -:response: UnfollowResponse +:response: AcknowledgedResponse -- [id="{upid}-{api}"] From d0b5006abc4634c00793a004875f43bee63f62b1 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 23 Nov 2018 06:58:05 +0000 Subject: [PATCH 14/62] [HLRC][ML] Add ML find file structure API (#35833) Relates to #29827 --- .../client/MLRequestConverters.java | 63 +++ .../client/MachineLearningClient.java | 43 ++ .../client/ml/FindFileStructureRequest.java | 302 ++++++++++ .../client/ml/FindFileStructureResponse.java | 70 +++ .../ml/filestructurefinder/FieldStats.java | 166 ++++++ .../ml/filestructurefinder/FileStructure.java | 516 ++++++++++++++++++ .../client/MLRequestConvertersTests.java | 83 +++ .../client/MachineLearningIT.java | 44 ++ .../MlClientDocumentationIT.java | 68 +++ .../ml/FindFileStructureRequestTests.java | 114 ++++ .../ml/FindFileStructureResponseTests.java | 49 ++ .../filestructurefinder/FieldStatsTests.java | 88 +++ .../FileStructureTests.java | 127 +++++ .../ml/find-file-structure.asciidoc | 53 ++ 14 files changed, 1786 insertions(+) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FileStructure.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FileStructureTests.java create mode 100644 docs/java-rest/high-level/ml/find-file-structure.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 57c40852078e0..5347b25c8fa42 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -36,6 +36,7 @@ import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; +import org.elasticsearch.client.ml.FindFileStructureRequest; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.GetBucketsRequest; @@ -70,6 +71,7 @@ import org.elasticsearch.client.ml.job.util.PageParams; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -648,4 +650,65 @@ static Request deleteFilter(DeleteFilterRequest deleteFilterRequest) { Request request = new Request(HttpDelete.METHOD_NAME, endpoint); return request; } + + static Request findFileStructure(FindFileStructureRequest findFileStructureRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("find_file_structure") + .build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + if (findFileStructureRequest.getLinesToSample() != null) { + params.putParam(FindFileStructureRequest.LINES_TO_SAMPLE.getPreferredName(), + findFileStructureRequest.getLinesToSample().toString()); + } + if (findFileStructureRequest.getTimeout() != null) { + params.putParam(FindFileStructureRequest.TIMEOUT.getPreferredName(), findFileStructureRequest.getTimeout().toString()); + } + if (findFileStructureRequest.getCharset() != null) { + params.putParam(FindFileStructureRequest.CHARSET.getPreferredName(), findFileStructureRequest.getCharset()); + } + if (findFileStructureRequest.getFormat() != null) { + params.putParam(FindFileStructureRequest.FORMAT.getPreferredName(), findFileStructureRequest.getFormat().toString()); + } + if (findFileStructureRequest.getColumnNames() != null) { + params.putParam(FindFileStructureRequest.COLUMN_NAMES.getPreferredName(), + Strings.collectionToCommaDelimitedString(findFileStructureRequest.getColumnNames())); + } + if (findFileStructureRequest.getHasHeaderRow() != null) { + params.putParam(FindFileStructureRequest.HAS_HEADER_ROW.getPreferredName(), + findFileStructureRequest.getHasHeaderRow().toString()); + } + if (findFileStructureRequest.getDelimiter() != null) { + params.putParam(FindFileStructureRequest.DELIMITER.getPreferredName(), + findFileStructureRequest.getDelimiter().toString()); + } + if (findFileStructureRequest.getQuote() != null) { + params.putParam(FindFileStructureRequest.QUOTE.getPreferredName(), findFileStructureRequest.getQuote().toString()); + } + if (findFileStructureRequest.getShouldTrimFields() != null) { + params.putParam(FindFileStructureRequest.SHOULD_TRIM_FIELDS.getPreferredName(), + findFileStructureRequest.getShouldTrimFields().toString()); + } + if (findFileStructureRequest.getGrokPattern() != null) { + params.putParam(FindFileStructureRequest.GROK_PATTERN.getPreferredName(), findFileStructureRequest.getGrokPattern()); + } + if (findFileStructureRequest.getTimestampFormat() != null) { + params.putParam(FindFileStructureRequest.TIMESTAMP_FORMAT.getPreferredName(), findFileStructureRequest.getTimestampFormat()); + } + if (findFileStructureRequest.getTimestampField() != null) { + params.putParam(FindFileStructureRequest.TIMESTAMP_FIELD.getPreferredName(), findFileStructureRequest.getTimestampField()); + } + if (findFileStructureRequest.getExplain() != null) { + params.putParam(FindFileStructureRequest.EXPLAIN.getPreferredName(), findFileStructureRequest.getExplain().toString()); + } + + BytesReference sample = findFileStructureRequest.getSample(); + BytesRef source = sample.toBytesRef(); + HttpEntity byteEntity = new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(XContentType.JSON)); + request.setEntity(byteEntity); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index 204dfaa87bc95..a4d8f1b9aa366 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -31,6 +31,8 @@ import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteJobResponse; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; +import org.elasticsearch.client.ml.FindFileStructureRequest; +import org.elasticsearch.client.ml.FindFileStructureResponse; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.ForecastJobRequest; @@ -1711,4 +1713,45 @@ public void deleteFilterAsync(DeleteFilterRequest request, RequestOptions option listener, Collections.emptySet()); } + + /** + * Finds the structure of a file + *

    + * For additional info + * see + * ML Find File Structure documentation + * + * @param request The find file structure request + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response containing details of the file structure + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public FindFileStructureResponse findFileStructure(FindFileStructureRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::findFileStructure, + options, + FindFileStructureResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Finds the structure of a file asynchronously and notifies the listener on completion + *

    + * For additional info + * see + * ML Find File Structure documentation + * + * @param request The find file structure request + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void findFileStructureAsync(FindFileStructureRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::findFileStructure, + options, + FindFileStructureResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java new file mode 100644 index 0000000000000..90e0c720e8811 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureRequest.java @@ -0,0 +1,302 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public class FindFileStructureRequest implements Validatable, ToXContent { + + public static final ParseField LINES_TO_SAMPLE = new ParseField("lines_to_sample"); + public static final ParseField TIMEOUT = new ParseField("timeout"); + public static final ParseField CHARSET = FileStructure.CHARSET; + public static final ParseField FORMAT = FileStructure.FORMAT; + public static final ParseField COLUMN_NAMES = FileStructure.COLUMN_NAMES; + public static final ParseField HAS_HEADER_ROW = FileStructure.HAS_HEADER_ROW; + public static final ParseField DELIMITER = FileStructure.DELIMITER; + public static final ParseField QUOTE = FileStructure.QUOTE; + public static final ParseField SHOULD_TRIM_FIELDS = FileStructure.SHOULD_TRIM_FIELDS; + public static final ParseField GROK_PATTERN = FileStructure.GROK_PATTERN; + // This one is plural in FileStructure, but singular in FileStructureOverrides + public static final ParseField TIMESTAMP_FORMAT = new ParseField("timestamp_format"); + public static final ParseField TIMESTAMP_FIELD = FileStructure.TIMESTAMP_FIELD; + public static final ParseField EXPLAIN = new ParseField("explain"); + + private Integer linesToSample; + private TimeValue timeout; + private String charset; + private FileStructure.Format format; + private List columnNames; + private Boolean hasHeaderRow; + private Character delimiter; + private Character quote; + private Boolean shouldTrimFields; + private String grokPattern; + private String timestampFormat; + private String timestampField; + private Boolean explain; + private BytesReference sample; + + public FindFileStructureRequest() { + } + + public Integer getLinesToSample() { + return linesToSample; + } + + public void setLinesToSample(Integer linesToSample) { + this.linesToSample = linesToSample; + } + + public TimeValue getTimeout() { + return timeout; + } + + public void setTimeout(TimeValue timeout) { + this.timeout = timeout; + } + + public String getCharset() { + return charset; + } + + public void setCharset(String charset) { + this.charset = (charset == null || charset.isEmpty()) ? null : charset; + } + + public FileStructure.Format getFormat() { + return format; + } + + public void setFormat(FileStructure.Format format) { + this.format = format; + } + + public void setFormat(String format) { + this.format = (format == null || format.isEmpty()) ? null : FileStructure.Format.fromString(format); + } + + public List getColumnNames() { + return columnNames; + } + + public void setColumnNames(List columnNames) { + this.columnNames = (columnNames == null || columnNames.isEmpty()) ? null : columnNames; + } + + public void setColumnNames(String[] columnNames) { + this.columnNames = (columnNames == null || columnNames.length == 0) ? null : Arrays.asList(columnNames); + } + + public Boolean getHasHeaderRow() { + return hasHeaderRow; + } + + public void setHasHeaderRow(Boolean hasHeaderRow) { + this.hasHeaderRow = hasHeaderRow; + } + + public Character getDelimiter() { + return delimiter; + } + + public void setDelimiter(Character delimiter) { + this.delimiter = delimiter; + } + + public void setDelimiter(String delimiter) { + if (delimiter == null || delimiter.isEmpty()) { + this.delimiter = null; + } else if (delimiter.length() == 1) { + this.delimiter = delimiter.charAt(0); + } else { + throw new IllegalArgumentException(DELIMITER.getPreferredName() + " must be a single character"); + } + } + + public Character getQuote() { + return quote; + } + + public void setQuote(Character quote) { + this.quote = quote; + } + + public void setQuote(String quote) { + if (quote == null || quote.isEmpty()) { + this.quote = null; + } else if (quote.length() == 1) { + this.quote = quote.charAt(0); + } else { + throw new IllegalArgumentException(QUOTE.getPreferredName() + " must be a single character"); + } + } + + public Boolean getShouldTrimFields() { + return shouldTrimFields; + } + + public void setShouldTrimFields(Boolean shouldTrimFields) { + this.shouldTrimFields = shouldTrimFields; + } + + public String getGrokPattern() { + return grokPattern; + } + + public void setGrokPattern(String grokPattern) { + this.grokPattern = (grokPattern == null || grokPattern.isEmpty()) ? null : grokPattern; + } + + public String getTimestampFormat() { + return timestampFormat; + } + + public void setTimestampFormat(String timestampFormat) { + this.timestampFormat = (timestampFormat == null || timestampFormat.isEmpty()) ? null : timestampFormat; + } + + public String getTimestampField() { + return timestampField; + } + + public void setTimestampField(String timestampField) { + this.timestampField = (timestampField == null || timestampField.isEmpty()) ? null : timestampField; + } + + public Boolean getExplain() { + return explain; + } + + public void setExplain(Boolean explain) { + this.explain = explain; + } + + public BytesReference getSample() { + return sample; + } + + public void setSample(byte[] sample) { + this.sample = new BytesArray(sample); + } + + public void setSample(BytesReference sample) { + this.sample = Objects.requireNonNull(sample); + } + + @Override + public Optional validate() { + ValidationException validationException = new ValidationException(); + if (sample == null || sample.length() == 0) { + validationException.addValidationError("sample must be specified"); + } + return validationException.validationErrors().isEmpty() ? Optional.empty() : Optional.of(validationException); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + if (linesToSample != null) { + builder.field(LINES_TO_SAMPLE.getPreferredName(), linesToSample); + } + if (timeout != null) { + builder.field(TIMEOUT.getPreferredName(), timeout); + } + if (charset != null) { + builder.field(CHARSET.getPreferredName(), charset); + } + if (format != null) { + builder.field(FORMAT.getPreferredName(), format); + } + if (columnNames != null) { + builder.field(COLUMN_NAMES.getPreferredName(), columnNames); + } + if (hasHeaderRow != null) { + builder.field(HAS_HEADER_ROW.getPreferredName(), hasHeaderRow); + } + if (delimiter != null) { + builder.field(DELIMITER.getPreferredName(), delimiter.toString()); + } + if (quote != null) { + builder.field(QUOTE.getPreferredName(), quote.toString()); + } + if (shouldTrimFields != null) { + builder.field(SHOULD_TRIM_FIELDS.getPreferredName(), shouldTrimFields); + } + if (grokPattern != null) { + builder.field(GROK_PATTERN.getPreferredName(), grokPattern); + } + if (timestampFormat != null) { + builder.field(TIMESTAMP_FORMAT.getPreferredName(), timestampFormat); + } + if (timestampField != null) { + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestampField); + } + if (explain != null) { + builder.field(EXPLAIN.getPreferredName(), explain); + } + // Sample is not included in the X-Content representation + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(linesToSample, timeout, charset, format, columnNames, hasHeaderRow, delimiter, grokPattern, timestampFormat, + timestampField, explain, sample); + } + + @Override + public boolean equals(Object other) { + + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + FindFileStructureRequest that = (FindFileStructureRequest) other; + return Objects.equals(this.linesToSample, that.linesToSample) && + Objects.equals(this.timeout, that.timeout) && + Objects.equals(this.charset, that.charset) && + Objects.equals(this.format, that.format) && + Objects.equals(this.columnNames, that.columnNames) && + Objects.equals(this.hasHeaderRow, that.hasHeaderRow) && + Objects.equals(this.delimiter, that.delimiter) && + Objects.equals(this.grokPattern, that.grokPattern) && + Objects.equals(this.timestampFormat, that.timestampFormat) && + Objects.equals(this.timestampField, that.timestampField) && + Objects.equals(this.explain, that.explain) && + Objects.equals(this.sample, that.sample); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureResponse.java new file mode 100644 index 0000000000000..dcb5b3d54e0b3 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/FindFileStructureResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class FindFileStructureResponse implements ToXContentObject { + + private final FileStructure fileStructure; + + FindFileStructureResponse(FileStructure fileStructure) { + this.fileStructure = Objects.requireNonNull(fileStructure); + } + + public static FindFileStructureResponse fromXContent(XContentParser parser) throws IOException { + return new FindFileStructureResponse(FileStructure.PARSER.parse(parser, null).build()); + } + + public FileStructure getFileStructure() { + return fileStructure; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + fileStructure.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(fileStructure); + } + + @Override + public boolean equals(Object other) { + + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + FindFileStructureResponse that = (FindFileStructureResponse) other; + return Objects.equals(fileStructure, that.fileStructure); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java new file mode 100644 index 0000000000000..4391d03f6d940 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FieldStats.java @@ -0,0 +1,166 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.filestructurefinder; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class FieldStats implements ToXContentObject { + + public static final ParseField COUNT = new ParseField("count"); + public static final ParseField CARDINALITY = new ParseField("cardinality"); + public static final ParseField MIN_VALUE = new ParseField("min_value"); + public static final ParseField MAX_VALUE = new ParseField("max_value"); + public static final ParseField MEAN_VALUE = new ParseField("mean_value"); + public static final ParseField MEDIAN_VALUE = new ParseField("median_value"); + public static final ParseField TOP_HITS = new ParseField("top_hits"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("field_stats", true, + a -> new FieldStats((long) a[0], (int) a[1], (Double) a[2], (Double) a[3], (Double) a[4], (Double) a[5], + (List>) a[6])); + + static { + PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), CARDINALITY); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MIN_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MAX_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEAN_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEDIAN_VALUE); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapOrdered(), TOP_HITS); + } + + private final long count; + private final int cardinality; + private final Double minValue; + private final Double maxValue; + private final Double meanValue; + private final Double medianValue; + private final List> topHits; + + FieldStats(long count, int cardinality, Double minValue, Double maxValue, Double meanValue, Double medianValue, + List> topHits) { + this.count = count; + this.cardinality = cardinality; + this.minValue = minValue; + this.maxValue = maxValue; + this.meanValue = meanValue; + this.medianValue = medianValue; + this.topHits = (topHits == null) ? Collections.emptyList() : Collections.unmodifiableList(topHits); + } + + public long getCount() { + return count; + } + + public int getCardinality() { + return cardinality; + } + + public Double getMinValue() { + return minValue; + } + + public Double getMaxValue() { + return maxValue; + } + + public Double getMeanValue() { + return meanValue; + } + + public Double getMedianValue() { + return medianValue; + } + + public List> getTopHits() { + return topHits; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + + builder.startObject(); + builder.field(COUNT.getPreferredName(), count); + builder.field(CARDINALITY.getPreferredName(), cardinality); + if (minValue != null) { + builder.field(MIN_VALUE.getPreferredName(), toIntegerIfInteger(minValue)); + } + if (maxValue != null) { + builder.field(MAX_VALUE.getPreferredName(), toIntegerIfInteger(maxValue)); + } + if (meanValue != null) { + builder.field(MEAN_VALUE.getPreferredName(), toIntegerIfInteger(meanValue)); + } + if (medianValue != null) { + builder.field(MEDIAN_VALUE.getPreferredName(), toIntegerIfInteger(medianValue)); + } + if (topHits.isEmpty() == false) { + builder.field(TOP_HITS.getPreferredName(), topHits); + } + builder.endObject(); + + return builder; + } + + static Number toIntegerIfInteger(double d) { + + if (d >= Integer.MIN_VALUE && d <= Integer.MAX_VALUE && Double.compare(d, StrictMath.rint(d)) == 0) { + return (int) d; + } + + return d; + } + + @Override + public int hashCode() { + + return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits); + } + + @Override + public boolean equals(Object other) { + + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + FieldStats that = (FieldStats) other; + return this.count == that.count && + this.cardinality == that.cardinality && + Objects.equals(this.minValue, that.minValue) && + Objects.equals(this.maxValue, that.maxValue) && + Objects.equals(this.meanValue, that.meanValue) && + Objects.equals(this.medianValue, that.medianValue) && + Objects.equals(this.topHits, that.topHits); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FileStructure.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FileStructure.java new file mode 100644 index 0000000000000..215c19a0b3b78 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/filestructurefinder/FileStructure.java @@ -0,0 +1,516 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.filestructurefinder; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * Stores the file format determined by Machine Learning. + */ +public class FileStructure implements ToXContentObject { + + public enum Format { + + NDJSON, XML, DELIMITED, SEMI_STRUCTURED_TEXT; + + public static Format fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final ParseField NUM_LINES_ANALYZED = new ParseField("num_lines_analyzed"); + public static final ParseField NUM_MESSAGES_ANALYZED = new ParseField("num_messages_analyzed"); + public static final ParseField SAMPLE_START = new ParseField("sample_start"); + public static final ParseField CHARSET = new ParseField("charset"); + public static final ParseField HAS_BYTE_ORDER_MARKER = new ParseField("has_byte_order_marker"); + public static final ParseField FORMAT = new ParseField("format"); + public static final ParseField MULTILINE_START_PATTERN = new ParseField("multiline_start_pattern"); + public static final ParseField EXCLUDE_LINES_PATTERN = new ParseField("exclude_lines_pattern"); + public static final ParseField COLUMN_NAMES = new ParseField("column_names"); + public static final ParseField HAS_HEADER_ROW = new ParseField("has_header_row"); + public static final ParseField DELIMITER = new ParseField("delimiter"); + public static final ParseField QUOTE = new ParseField("quote"); + public static final ParseField SHOULD_TRIM_FIELDS = new ParseField("should_trim_fields"); + public static final ParseField GROK_PATTERN = new ParseField("grok_pattern"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp_field"); + public static final ParseField JODA_TIMESTAMP_FORMATS = new ParseField("joda_timestamp_formats"); + public static final ParseField JAVA_TIMESTAMP_FORMATS = new ParseField("java_timestamp_formats"); + public static final ParseField NEED_CLIENT_TIMEZONE = new ParseField("need_client_timezone"); + public static final ParseField MAPPINGS = new ParseField("mappings"); + public static final ParseField INGEST_PIPELINE = new ParseField("ingest_pipeline"); + public static final ParseField FIELD_STATS = new ParseField("field_stats"); + public static final ParseField EXPLANATION = new ParseField("explanation"); + + public static final ObjectParser PARSER = new ObjectParser<>("file_structure", true, Builder::new); + + static { + PARSER.declareInt(Builder::setNumLinesAnalyzed, NUM_LINES_ANALYZED); + PARSER.declareInt(Builder::setNumMessagesAnalyzed, NUM_MESSAGES_ANALYZED); + PARSER.declareString(Builder::setSampleStart, SAMPLE_START); + PARSER.declareString(Builder::setCharset, CHARSET); + PARSER.declareBoolean(Builder::setHasByteOrderMarker, HAS_BYTE_ORDER_MARKER); + PARSER.declareString((p, c) -> p.setFormat(Format.fromString(c)), FORMAT); + PARSER.declareString(Builder::setMultilineStartPattern, MULTILINE_START_PATTERN); + PARSER.declareString(Builder::setExcludeLinesPattern, EXCLUDE_LINES_PATTERN); + PARSER.declareStringArray(Builder::setColumnNames, COLUMN_NAMES); + PARSER.declareBoolean(Builder::setHasHeaderRow, HAS_HEADER_ROW); + PARSER.declareString((p, c) -> p.setDelimiter(c.charAt(0)), DELIMITER); + PARSER.declareString((p, c) -> p.setQuote(c.charAt(0)), QUOTE); + PARSER.declareBoolean(Builder::setShouldTrimFields, SHOULD_TRIM_FIELDS); + PARSER.declareString(Builder::setGrokPattern, GROK_PATTERN); + PARSER.declareString(Builder::setTimestampField, TIMESTAMP_FIELD); + PARSER.declareStringArray(Builder::setJodaTimestampFormats, JODA_TIMESTAMP_FORMATS); + PARSER.declareStringArray(Builder::setJavaTimestampFormats, JAVA_TIMESTAMP_FORMATS); + PARSER.declareBoolean(Builder::setNeedClientTimezone, NEED_CLIENT_TIMEZONE); + PARSER.declareObject(Builder::setMappings, (p, c) -> new TreeMap<>(p.map()), MAPPINGS); + PARSER.declareObject(Builder::setIngestPipeline, (p, c) -> p.mapOrdered(), INGEST_PIPELINE); + PARSER.declareObject(Builder::setFieldStats, (p, c) -> { + Map fieldStats = new TreeMap<>(); + while (p.nextToken() == XContentParser.Token.FIELD_NAME) { + fieldStats.put(p.currentName(), FieldStats.PARSER.apply(p, c)); + } + return fieldStats; + }, FIELD_STATS); + PARSER.declareStringArray(Builder::setExplanation, EXPLANATION); + } + + private final int numLinesAnalyzed; + private final int numMessagesAnalyzed; + private final String sampleStart; + private final String charset; + private final Boolean hasByteOrderMarker; + private final Format format; + private final String multilineStartPattern; + private final String excludeLinesPattern; + private final List columnNames; + private final Boolean hasHeaderRow; + private final Character delimiter; + private final Character quote; + private final Boolean shouldTrimFields; + private final String grokPattern; + private final List jodaTimestampFormats; + private final List javaTimestampFormats; + private final String timestampField; + private final boolean needClientTimezone; + private final SortedMap mappings; + private final Map ingestPipeline; + private final SortedMap fieldStats; + private final List explanation; + + private FileStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker, + Format format, String multilineStartPattern, String excludeLinesPattern, List columnNames, + Boolean hasHeaderRow, Character delimiter, Character quote, Boolean shouldTrimFields, String grokPattern, + String timestampField, List jodaTimestampFormats, List javaTimestampFormats, + boolean needClientTimezone, Map mappings, Map ingestPipeline, + Map fieldStats, List explanation) { + + this.numLinesAnalyzed = numLinesAnalyzed; + this.numMessagesAnalyzed = numMessagesAnalyzed; + this.sampleStart = Objects.requireNonNull(sampleStart); + this.charset = Objects.requireNonNull(charset); + this.hasByteOrderMarker = hasByteOrderMarker; + this.format = Objects.requireNonNull(format); + this.multilineStartPattern = multilineStartPattern; + this.excludeLinesPattern = excludeLinesPattern; + this.columnNames = (columnNames == null) ? null : Collections.unmodifiableList(new ArrayList<>(columnNames)); + this.hasHeaderRow = hasHeaderRow; + this.delimiter = delimiter; + this.quote = quote; + this.shouldTrimFields = shouldTrimFields; + this.grokPattern = grokPattern; + this.timestampField = timestampField; + this.jodaTimestampFormats = + (jodaTimestampFormats == null) ? null : Collections.unmodifiableList(new ArrayList<>(jodaTimestampFormats)); + this.javaTimestampFormats = + (javaTimestampFormats == null) ? null : Collections.unmodifiableList(new ArrayList<>(javaTimestampFormats)); + this.needClientTimezone = needClientTimezone; + this.mappings = Collections.unmodifiableSortedMap(new TreeMap<>(mappings)); + this.ingestPipeline = (ingestPipeline == null) ? null : Collections.unmodifiableMap(new LinkedHashMap<>(ingestPipeline)); + this.fieldStats = Collections.unmodifiableSortedMap(new TreeMap<>(fieldStats)); + this.explanation = (explanation == null) ? null : Collections.unmodifiableList(new ArrayList<>(explanation)); + } + + public int getNumLinesAnalyzed() { + return numLinesAnalyzed; + } + + public int getNumMessagesAnalyzed() { + return numMessagesAnalyzed; + } + + public String getSampleStart() { + return sampleStart; + } + + public String getCharset() { + return charset; + } + + public Boolean getHasByteOrderMarker() { + return hasByteOrderMarker; + } + + public Format getFormat() { + return format; + } + + public String getMultilineStartPattern() { + return multilineStartPattern; + } + + public String getExcludeLinesPattern() { + return excludeLinesPattern; + } + + public List getColumnNames() { + return columnNames; + } + + public Boolean getHasHeaderRow() { + return hasHeaderRow; + } + + public Character getDelimiter() { + return delimiter; + } + + public Character getQuote() { + return quote; + } + + public Boolean getShouldTrimFields() { + return shouldTrimFields; + } + + public String getGrokPattern() { + return grokPattern; + } + + public String getTimestampField() { + return timestampField; + } + + public List getJodaTimestampFormats() { + return jodaTimestampFormats; + } + + public List getJavaTimestampFormats() { + return javaTimestampFormats; + } + + public boolean needClientTimezone() { + return needClientTimezone; + } + + public SortedMap getMappings() { + return mappings; + } + + public Map getIngestPipeline() { + return ingestPipeline; + } + + public SortedMap getFieldStats() { + return fieldStats; + } + + public List getExplanation() { + return explanation; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + builder.startObject(); + builder.field(NUM_LINES_ANALYZED.getPreferredName(), numLinesAnalyzed); + builder.field(NUM_MESSAGES_ANALYZED.getPreferredName(), numMessagesAnalyzed); + builder.field(SAMPLE_START.getPreferredName(), sampleStart); + builder.field(CHARSET.getPreferredName(), charset); + if (hasByteOrderMarker != null) { + builder.field(HAS_BYTE_ORDER_MARKER.getPreferredName(), hasByteOrderMarker.booleanValue()); + } + builder.field(FORMAT.getPreferredName(), format); + if (multilineStartPattern != null && multilineStartPattern.isEmpty() == false) { + builder.field(MULTILINE_START_PATTERN.getPreferredName(), multilineStartPattern); + } + if (excludeLinesPattern != null && excludeLinesPattern.isEmpty() == false) { + builder.field(EXCLUDE_LINES_PATTERN.getPreferredName(), excludeLinesPattern); + } + if (columnNames != null && columnNames.isEmpty() == false) { + builder.field(COLUMN_NAMES.getPreferredName(), columnNames); + } + if (hasHeaderRow != null) { + builder.field(HAS_HEADER_ROW.getPreferredName(), hasHeaderRow.booleanValue()); + } + if (delimiter != null) { + builder.field(DELIMITER.getPreferredName(), String.valueOf(delimiter)); + } + if (quote != null) { + builder.field(QUOTE.getPreferredName(), String.valueOf(quote)); + } + if (shouldTrimFields != null) { + builder.field(SHOULD_TRIM_FIELDS.getPreferredName(), shouldTrimFields.booleanValue()); + } + if (grokPattern != null && grokPattern.isEmpty() == false) { + builder.field(GROK_PATTERN.getPreferredName(), grokPattern); + } + if (timestampField != null && timestampField.isEmpty() == false) { + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestampField); + } + if (jodaTimestampFormats != null && jodaTimestampFormats.isEmpty() == false) { + builder.field(JODA_TIMESTAMP_FORMATS.getPreferredName(), jodaTimestampFormats); + } + if (javaTimestampFormats != null && javaTimestampFormats.isEmpty() == false) { + builder.field(JAVA_TIMESTAMP_FORMATS.getPreferredName(), javaTimestampFormats); + } + builder.field(NEED_CLIENT_TIMEZONE.getPreferredName(), needClientTimezone); + builder.field(MAPPINGS.getPreferredName(), mappings); + if (ingestPipeline != null) { + builder.field(INGEST_PIPELINE.getPreferredName(), ingestPipeline); + } + if (fieldStats.isEmpty() == false) { + builder.startObject(FIELD_STATS.getPreferredName()); + for (Map.Entry entry : fieldStats.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + if (explanation != null && explanation.isEmpty() == false) { + builder.field(EXPLANATION.getPreferredName(), explanation); + } + builder.endObject(); + + return builder; + } + + @Override + public int hashCode() { + + return Objects.hash(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, + multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern, + timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, fieldStats, explanation); + } + + @Override + public boolean equals(Object other) { + + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + FileStructure that = (FileStructure) other; + return this.numLinesAnalyzed == that.numLinesAnalyzed && + this.numMessagesAnalyzed == that.numMessagesAnalyzed && + Objects.equals(this.sampleStart, that.sampleStart) && + Objects.equals(this.charset, that.charset) && + Objects.equals(this.hasByteOrderMarker, that.hasByteOrderMarker) && + Objects.equals(this.format, that.format) && + Objects.equals(this.multilineStartPattern, that.multilineStartPattern) && + Objects.equals(this.excludeLinesPattern, that.excludeLinesPattern) && + Objects.equals(this.columnNames, that.columnNames) && + Objects.equals(this.hasHeaderRow, that.hasHeaderRow) && + Objects.equals(this.delimiter, that.delimiter) && + Objects.equals(this.quote, that.quote) && + Objects.equals(this.shouldTrimFields, that.shouldTrimFields) && + Objects.equals(this.grokPattern, that.grokPattern) && + Objects.equals(this.timestampField, that.timestampField) && + Objects.equals(this.jodaTimestampFormats, that.jodaTimestampFormats) && + Objects.equals(this.javaTimestampFormats, that.javaTimestampFormats) && + this.needClientTimezone == that.needClientTimezone && + Objects.equals(this.mappings, that.mappings) && + Objects.equals(this.fieldStats, that.fieldStats) && + Objects.equals(this.explanation, that.explanation); + } + + public static class Builder { + + private int numLinesAnalyzed; + private int numMessagesAnalyzed; + private String sampleStart; + private String charset; + private Boolean hasByteOrderMarker; + private Format format; + private String multilineStartPattern; + private String excludeLinesPattern; + private List columnNames; + private Boolean hasHeaderRow; + private Character delimiter; + private Character quote; + private Boolean shouldTrimFields; + private String grokPattern; + private String timestampField; + private List jodaTimestampFormats; + private List javaTimestampFormats; + private boolean needClientTimezone; + private Map mappings = Collections.emptyMap(); + private Map ingestPipeline; + private Map fieldStats = Collections.emptyMap(); + private List explanation; + + Builder() { + this(Format.SEMI_STRUCTURED_TEXT); + } + + Builder(Format format) { + setFormat(format); + } + + Builder setNumLinesAnalyzed(int numLinesAnalyzed) { + this.numLinesAnalyzed = numLinesAnalyzed; + return this; + } + + Builder setNumMessagesAnalyzed(int numMessagesAnalyzed) { + this.numMessagesAnalyzed = numMessagesAnalyzed; + return this; + } + + Builder setSampleStart(String sampleStart) { + this.sampleStart = Objects.requireNonNull(sampleStart); + return this; + } + + Builder setCharset(String charset) { + this.charset = Objects.requireNonNull(charset); + return this; + } + + Builder setHasByteOrderMarker(Boolean hasByteOrderMarker) { + this.hasByteOrderMarker = hasByteOrderMarker; + return this; + } + + Builder setFormat(Format format) { + this.format = Objects.requireNonNull(format); + return this; + } + + Builder setMultilineStartPattern(String multilineStartPattern) { + this.multilineStartPattern = multilineStartPattern; + return this; + } + + Builder setExcludeLinesPattern(String excludeLinesPattern) { + this.excludeLinesPattern = excludeLinesPattern; + return this; + } + + Builder setColumnNames(List columnNames) { + this.columnNames = columnNames; + return this; + } + + Builder setHasHeaderRow(Boolean hasHeaderRow) { + this.hasHeaderRow = hasHeaderRow; + return this; + } + + Builder setDelimiter(Character delimiter) { + this.delimiter = delimiter; + return this; + } + + Builder setQuote(Character quote) { + this.quote = quote; + return this; + } + + Builder setShouldTrimFields(Boolean shouldTrimFields) { + this.shouldTrimFields = shouldTrimFields; + return this; + } + + Builder setGrokPattern(String grokPattern) { + this.grokPattern = grokPattern; + return this; + } + + Builder setTimestampField(String timestampField) { + this.timestampField = timestampField; + return this; + } + + Builder setJodaTimestampFormats(List jodaTimestampFormats) { + this.jodaTimestampFormats = jodaTimestampFormats; + return this; + } + + Builder setJavaTimestampFormats(List javaTimestampFormats) { + this.javaTimestampFormats = javaTimestampFormats; + return this; + } + + Builder setNeedClientTimezone(boolean needClientTimezone) { + this.needClientTimezone = needClientTimezone; + return this; + } + + Builder setMappings(Map mappings) { + this.mappings = Objects.requireNonNull(mappings); + return this; + } + + Builder setIngestPipeline(Map ingestPipeline) { + this.ingestPipeline = ingestPipeline; + return this; + } + + Builder setFieldStats(Map fieldStats) { + this.fieldStats = Objects.requireNonNull(fieldStats); + return this; + } + + Builder setExplanation(List explanation) { + this.explanation = explanation; + return this; + } + + public FileStructure build() { + + return new FileStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, + multilineStartPattern, excludeLinesPattern, columnNames, hasHeaderRow, delimiter, quote, shouldTrimFields, grokPattern, + timestampField, jodaTimestampFormats, javaTimestampFormats, needClientTimezone, mappings, ingestPipeline, fieldStats, + explanation); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index f5c421c329fca..1685c62a2963f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -32,6 +32,8 @@ import org.elasticsearch.client.ml.DeleteForecastRequest; import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; +import org.elasticsearch.client.ml.FindFileStructureRequest; +import org.elasticsearch.client.ml.FindFileStructureRequestTests; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.ForecastJobRequest; import org.elasticsearch.client.ml.GetBucketsRequest; @@ -69,6 +71,7 @@ import org.elasticsearch.client.ml.calendars.ScheduledEventTests; import org.elasticsearch.client.ml.datafeed.DatafeedConfig; import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.Detector; import org.elasticsearch.client.ml.job.config.Job; @@ -87,6 +90,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -715,6 +719,85 @@ public void testDeleteFilter() { assertNull(request.getEntity()); } + public void testFindFileStructure() throws Exception { + + String sample = randomAlphaOfLength(randomIntBetween(1000, 2000)); + FindFileStructureRequest findFileStructureRequest = FindFileStructureRequestTests.createTestRequestWithoutSample(); + findFileStructureRequest.setSample(sample.getBytes(StandardCharsets.UTF_8)); + Request request = MLRequestConverters.findFileStructure(findFileStructureRequest); + + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/find_file_structure", request.getEndpoint()); + if (findFileStructureRequest.getLinesToSample() != null) { + assertEquals(findFileStructureRequest.getLinesToSample(), Integer.valueOf(request.getParameters().get("lines_to_sample"))); + } else { + assertNull(request.getParameters().get("lines_to_sample")); + } + if (findFileStructureRequest.getTimeout() != null) { + assertEquals(findFileStructureRequest.getTimeout().toString(), request.getParameters().get("timeout")); + } else { + assertNull(request.getParameters().get("timeout")); + } + if (findFileStructureRequest.getCharset() != null) { + assertEquals(findFileStructureRequest.getCharset(), request.getParameters().get("charset")); + } else { + assertNull(request.getParameters().get("charset")); + } + if (findFileStructureRequest.getFormat() != null) { + assertEquals(findFileStructureRequest.getFormat(), FileStructure.Format.fromString(request.getParameters().get("format"))); + } else { + assertNull(request.getParameters().get("format")); + } + if (findFileStructureRequest.getColumnNames() != null) { + assertEquals(findFileStructureRequest.getColumnNames(), + Arrays.asList(Strings.splitStringByCommaToArray(request.getParameters().get("column_names")))); + } else { + assertNull(request.getParameters().get("column_names")); + } + if (findFileStructureRequest.getHasHeaderRow() != null) { + assertEquals(findFileStructureRequest.getHasHeaderRow(), Boolean.valueOf(request.getParameters().get("has_header_row"))); + } else { + assertNull(request.getParameters().get("has_header_row")); + } + if (findFileStructureRequest.getDelimiter() != null) { + assertEquals(findFileStructureRequest.getDelimiter().toString(), request.getParameters().get("delimiter")); + } else { + assertNull(request.getParameters().get("delimiter")); + } + if (findFileStructureRequest.getQuote() != null) { + assertEquals(findFileStructureRequest.getQuote().toString(), request.getParameters().get("quote")); + } else { + assertNull(request.getParameters().get("quote")); + } + if (findFileStructureRequest.getShouldTrimFields() != null) { + assertEquals(findFileStructureRequest.getShouldTrimFields(), + Boolean.valueOf(request.getParameters().get("should_trim_fields"))); + } else { + assertNull(request.getParameters().get("should_trim_fields")); + } + if (findFileStructureRequest.getGrokPattern() != null) { + assertEquals(findFileStructureRequest.getGrokPattern(), request.getParameters().get("grok_pattern")); + } else { + assertNull(request.getParameters().get("grok_pattern")); + } + if (findFileStructureRequest.getTimestampFormat() != null) { + assertEquals(findFileStructureRequest.getTimestampFormat(), request.getParameters().get("timestamp_format")); + } else { + assertNull(request.getParameters().get("timestamp_format")); + } + if (findFileStructureRequest.getTimestampField() != null) { + assertEquals(findFileStructureRequest.getTimestampField(), request.getParameters().get("timestamp_field")); + } else { + assertNull(request.getParameters().get("timestamp_field")); + } + if (findFileStructureRequest.getExplain() != null) { + assertEquals(findFileStructureRequest.getExplain(), Boolean.valueOf(request.getParameters().get("explain"))); + } else { + assertNull(request.getParameters().get("explain")); + } + assertEquals(sample, requestEntityToString(request)); + } + private static Job createValidJob(String jobId) { AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList( Detector.builder().setFunction("count").build())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index fe8d5d96b5902..66f9acc065e3c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -38,6 +38,8 @@ import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteJobResponse; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; +import org.elasticsearch.client.ml.FindFileStructureRequest; +import org.elasticsearch.client.ml.FindFileStructureResponse; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.ForecastJobRequest; @@ -94,6 +96,7 @@ import org.elasticsearch.client.ml.datafeed.DatafeedState; import org.elasticsearch.client.ml.datafeed.DatafeedStats; import org.elasticsearch.client.ml.datafeed.DatafeedUpdate; +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.DataDescription; import org.elasticsearch.client.ml.job.config.Detector; @@ -110,11 +113,13 @@ import org.junit.After; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -1306,4 +1311,43 @@ public void testRevertModelSnapshot() throws IOException { assertEquals(snapshotId, model.getSnapshotId()); } } + + public void testFindFileStructure() throws IOException { + + String sample = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," + + "\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," + + "\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261151445," + + "\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," + + "\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n"; + + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + FindFileStructureRequest request = new FindFileStructureRequest(); + request.setSample(sample.getBytes(StandardCharsets.UTF_8)); + + FindFileStructureResponse response = + execute(request, machineLearningClient::findFileStructure, machineLearningClient::findFileStructureAsync); + + FileStructure structure = response.getFileStructure(); + + assertEquals(2, structure.getNumLinesAnalyzed()); + assertEquals(2, structure.getNumMessagesAnalyzed()); + assertEquals(sample, structure.getSampleStart()); + assertEquals(FileStructure.Format.NDJSON, structure.getFormat()); + assertEquals(StandardCharsets.UTF_8.displayName(Locale.ROOT), structure.getCharset()); + assertFalse(structure.getHasByteOrderMarker()); + assertNull(structure.getMultilineStartPattern()); + assertNull(structure.getExcludeLinesPattern()); + assertNull(structure.getColumnNames()); + assertNull(structure.getHasHeaderRow()); + assertNull(structure.getDelimiter()); + assertNull(structure.getQuote()); + assertNull(structure.getShouldTrimFields()); + assertNull(structure.getGrokPattern()); + assertEquals(Collections.singletonList("UNIX_MS"), structure.getJavaTimestampFormats()); + assertEquals(Collections.singletonList("UNIX_MS"), structure.getJodaTimestampFormats()); + assertEquals("timestamp", structure.getTimestampField()); + assertFalse(structure.needClientTimezone()); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 17f5bcffbccb4..eea4ddc2fd610 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -44,6 +44,8 @@ import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteJobResponse; import org.elasticsearch.client.ml.DeleteModelSnapshotRequest; +import org.elasticsearch.client.ml.FindFileStructureRequest; +import org.elasticsearch.client.ml.FindFileStructureResponse; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.ForecastJobRequest; @@ -110,6 +112,7 @@ import org.elasticsearch.client.ml.datafeed.DatafeedStats; import org.elasticsearch.client.ml.datafeed.DatafeedUpdate; import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig; +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.AnalysisLimits; import org.elasticsearch.client.ml.job.config.DataDescription; @@ -140,6 +143,9 @@ import org.junit.After; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -1730,6 +1736,68 @@ public void onFailure(Exception e) { } } + public void testFindFileStructure() throws Exception { + RestHighLevelClient client = highLevelClient(); + + Path anInterestingFile = createTempFile(); + String contents = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\"," + + "\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 1\",\"class\":\"ml\"," + + "\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261151445," + + "\"level\":\"INFO\",\"pid\":42,\"thread\":\"0x7fff7d2a8000\",\"message\":\"message 2\",\"class\":\"ml\"," + + "\"method\":\"core::SomeNoiseMaker\",\"file\":\"Noisemaker.cc\",\"line\":333}\n"; + Files.write(anInterestingFile, Collections.singleton(contents), StandardCharsets.UTF_8); + + { + // tag::find-file-structure-request + FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest(); // <1> + findFileStructureRequest.setSample(Files.readAllBytes(anInterestingFile)); // <2> + // end::find-file-structure-request + + // tag::find-file-structure-request-options + findFileStructureRequest.setLinesToSample(500); // <1> + findFileStructureRequest.setExplain(true); // <2> + // end::find-file-structure-request-options + + // tag::find-file-structure-execute + FindFileStructureResponse findFileStructureResponse = + client.machineLearning().findFileStructure(findFileStructureRequest, RequestOptions.DEFAULT); + // end::find-file-structure-execute + + // tag::find-file-structure-response + FileStructure structure = findFileStructureResponse.getFileStructure(); // <1> + // end::find-file-structure-response + assertEquals(2, structure.getNumLinesAnalyzed()); + } + { + // tag::find-file-structure-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(FindFileStructureResponse findFileStructureResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::find-file-structure-execute-listener + FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest(); + findFileStructureRequest.setSample(Files.readAllBytes(anInterestingFile)); + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::find-file-structure-execute-async + client.machineLearning().findFileStructureAsync(findFileStructureRequest, RequestOptions.DEFAULT, listener); // <1> + // end::find-file-structure-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testGetInfluencers() throws IOException, InterruptedException { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java new file mode 100644 index 0000000000000..4cb8bf0a7c166 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureRequestTests.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.filestructurefinder.FileStructure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.Arrays; + +public class FindFileStructureRequestTests extends AbstractXContentTestCase { + + private static final ObjectParser PARSER = + new ObjectParser<>("find_file_structure_request", FindFileStructureRequest::new); + + static { + PARSER.declareInt(FindFileStructureRequest::setLinesToSample, FindFileStructureRequest.LINES_TO_SAMPLE); + PARSER.declareString((p, c) -> p.setTimeout(TimeValue.parseTimeValue(c, FindFileStructureRequest.TIMEOUT.getPreferredName())), + FindFileStructureRequest.TIMEOUT); + PARSER.declareString(FindFileStructureRequest::setCharset, FindFileStructureRequest.CHARSET); + PARSER.declareString(FindFileStructureRequest::setFormat, FindFileStructureRequest.FORMAT); + PARSER.declareStringArray(FindFileStructureRequest::setColumnNames, FindFileStructureRequest.COLUMN_NAMES); + PARSER.declareBoolean(FindFileStructureRequest::setHasHeaderRow, FindFileStructureRequest.HAS_HEADER_ROW); + PARSER.declareString(FindFileStructureRequest::setDelimiter, FindFileStructureRequest.DELIMITER); + PARSER.declareString(FindFileStructureRequest::setQuote, FindFileStructureRequest.QUOTE); + PARSER.declareBoolean(FindFileStructureRequest::setShouldTrimFields, FindFileStructureRequest.SHOULD_TRIM_FIELDS); + PARSER.declareString(FindFileStructureRequest::setGrokPattern, FindFileStructureRequest.GROK_PATTERN); + PARSER.declareString(FindFileStructureRequest::setTimestampFormat, FindFileStructureRequest.TIMESTAMP_FORMAT); + PARSER.declareString(FindFileStructureRequest::setTimestampField, FindFileStructureRequest.TIMESTAMP_FIELD); + PARSER.declareBoolean(FindFileStructureRequest::setExplain, FindFileStructureRequest.EXPLAIN); + // Sample is not included in the X-Content representation + } + + @Override + protected FindFileStructureRequest doParseInstance(XContentParser parser) throws IOException { + return PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected FindFileStructureRequest createTestInstance() { + return createTestRequestWithoutSample(); + } + + public static FindFileStructureRequest createTestRequestWithoutSample() { + + FindFileStructureRequest findFileStructureRequest = new FindFileStructureRequest(); + if (randomBoolean()) { + findFileStructureRequest.setLinesToSample(randomIntBetween(1000, 2000)); + } + if (randomBoolean()) { + findFileStructureRequest.setTimeout(TimeValue.timeValueSeconds(randomIntBetween(10, 20))); + } + if (randomBoolean()) { + findFileStructureRequest.setCharset(Charset.defaultCharset().toString()); + } + if (randomBoolean()) { + findFileStructureRequest.setFormat(randomFrom(FileStructure.Format.values())); + } + if (randomBoolean()) { + findFileStructureRequest.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false))); + } + if (randomBoolean()) { + findFileStructureRequest.setHasHeaderRow(randomBoolean()); + } + if (randomBoolean()) { + findFileStructureRequest.setDelimiter(randomAlphaOfLength(1)); + } + if (randomBoolean()) { + findFileStructureRequest.setQuote(randomAlphaOfLength(1)); + } + if (randomBoolean()) { + findFileStructureRequest.setShouldTrimFields(randomBoolean()); + } + if (randomBoolean()) { + findFileStructureRequest.setGrokPattern(randomAlphaOfLength(100)); + } + if (randomBoolean()) { + findFileStructureRequest.setTimestampFormat(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + findFileStructureRequest.setTimestampField(randomAlphaOfLength(10)); + } + if (randomBoolean()) { + findFileStructureRequest.setExplain(randomBoolean()); + } + + return findFileStructureRequest; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureResponseTests.java new file mode 100644 index 0000000000000..3d8b778feea8a --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/FindFileStructureResponseTests.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.filestructurefinder.FileStructureTests; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.function.Predicate; + +public class FindFileStructureResponseTests extends AbstractXContentTestCase { + + @Override + protected FindFileStructureResponse createTestInstance() { + return new FindFileStructureResponse(FileStructureTests.createTestFileStructure()); + } + + @Override + protected FindFileStructureResponse doParseInstance(XContentParser parser) throws IOException { + return FindFileStructureResponse.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> !field.isEmpty(); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java new file mode 100644 index 0000000000000..daf6c4af90ddc --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FieldStatsTests.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.filestructurefinder; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +public class FieldStatsTests extends AbstractXContentTestCase { + + @Override + protected FieldStats createTestInstance() { + return createTestFieldStats(); + } + + static FieldStats createTestFieldStats() { + + long count = randomIntBetween(1, 100000); + int cardinality = randomIntBetween(1, (int) count); + + Double minValue = null; + Double maxValue = null; + Double meanValue = null; + Double medianValue = null; + boolean isMetric = randomBoolean(); + if (isMetric) { + if (randomBoolean()) { + minValue = randomDouble(); + maxValue = randomDouble(); + } else { + minValue = (double) randomInt(); + maxValue = (double) randomInt(); + } + meanValue = randomDouble(); + medianValue = randomDouble(); + } + + List> topHits = new ArrayList<>(); + for (int i = 0; i < Math.min(10, cardinality); ++i) { + Map topHit = new LinkedHashMap<>(); + if (isMetric) { + topHit.put("value", randomBoolean() ? randomDouble() : (double) randomInt()); + } else { + topHit.put("value", randomAlphaOfLength(20)); + } + topHit.put("count", randomIntBetween(1, cardinality)); + topHits.add(topHit); + } + + return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits); + } + + @Override + protected FieldStats doParseInstance(XContentParser parser) { + return FieldStats.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.contains(FieldStats.TOP_HITS.getPreferredName()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FileStructureTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FileStructureTests.java new file mode 100644 index 0000000000000..fe7c1bce214e4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/filestructurefinder/FileStructureTests.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.filestructurefinder; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Predicate; + +public class FileStructureTests extends AbstractXContentTestCase { + + @Override + protected FileStructure createTestInstance() { + return createTestFileStructure(); + } + + public static FileStructure createTestFileStructure() { + + FileStructure.Format format = randomFrom(EnumSet.allOf(FileStructure.Format.class)); + + FileStructure.Builder builder = new FileStructure.Builder(format); + + int numLinesAnalyzed = randomIntBetween(2, 10000); + builder.setNumLinesAnalyzed(numLinesAnalyzed); + int numMessagesAnalyzed = randomIntBetween(1, numLinesAnalyzed); + builder.setNumMessagesAnalyzed(numMessagesAnalyzed); + builder.setSampleStart(randomAlphaOfLength(1000)); + + String charset = randomFrom(Charset.availableCharsets().keySet()); + builder.setCharset(charset); + if (charset.toUpperCase(Locale.ROOT).startsWith("UTF")) { + builder.setHasByteOrderMarker(randomBoolean()); + } + + if (numMessagesAnalyzed < numLinesAnalyzed) { + builder.setMultilineStartPattern(randomAlphaOfLength(100)); + } + if (randomBoolean()) { + builder.setExcludeLinesPattern(randomAlphaOfLength(100)); + } + + if (format == FileStructure.Format.DELIMITED) { + builder.setColumnNames(Arrays.asList(generateRandomStringArray(10, 10, false, false))); + builder.setHasHeaderRow(randomBoolean()); + builder.setDelimiter(randomFrom(',', '\t', ';', '|')); + builder.setQuote(randomFrom('"', '\'')); + } + + if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT) { + builder.setGrokPattern(randomAlphaOfLength(100)); + } + + if (format == FileStructure.Format.SEMI_STRUCTURED_TEXT || randomBoolean()) { + builder.setTimestampField(randomAlphaOfLength(10)); + builder.setJodaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); + builder.setJavaTimestampFormats(Arrays.asList(generateRandomStringArray(3, 20, false, false))); + builder.setNeedClientTimezone(randomBoolean()); + } + + Map mappings = new TreeMap<>(); + for (String field : generateRandomStringArray(5, 20, false, false)) { + mappings.put(field, Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(10))); + } + builder.setMappings(mappings); + + if (randomBoolean()) { + Map ingestPipeline = new LinkedHashMap<>(); + for (String field : generateRandomStringArray(5, 20, false, false)) { + ingestPipeline.put(field, Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(10))); + } + builder.setMappings(ingestPipeline); + } + + if (randomBoolean()) { + Map fieldStats = new TreeMap<>(); + for (String field : generateRandomStringArray(5, 20, false, false)) { + fieldStats.put(field, FieldStatsTests.createTestFieldStats()); + } + builder.setFieldStats(fieldStats); + } + + builder.setExplanation(Arrays.asList(generateRandomStringArray(10, 150, false, false))); + + return builder.build(); + } + + @Override + protected FileStructure doParseInstance(XContentParser parser) { + return FileStructure.PARSER.apply(parser, null).build(); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // unknown fields are only guaranteed to be ignored at the top level - below this several data + // structures (e.g. mappings, ingest pipeline, field stats) will preserve arbitrary fields + return field -> !field.isEmpty(); + } +} diff --git a/docs/java-rest/high-level/ml/find-file-structure.asciidoc b/docs/java-rest/high-level/ml/find-file-structure.asciidoc new file mode 100644 index 0000000000000..5882fc0fce2e2 --- /dev/null +++ b/docs/java-rest/high-level/ml/find-file-structure.asciidoc @@ -0,0 +1,53 @@ +-- +:api: find-file-structure +:request: FindFileStructureRequest +:response: FindFileStructureResponse +-- +[id="{upid}-{api}"] +=== Find File Structure API + +The Find File Structure API can be used to find the structure of a text file +and other information that will be useful to import its contents to an {es} +index. It accepts a +{request}+ object and responds +with a +{response}+ object. + +[id="{upid}-{api}-request"] +==== Find File Structure Request + +A sample from the beginning of the file (or the entire file contents if +it's small) must be added to the +{request}+ object using the +`FindFileStructureRequest#setSample` method. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Create a new `FindFileStructureRequest` object +<2> Add the contents of `anInterestingFile` to the request + +==== Optional Arguments + +The following arguments are optional. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-options] +-------------------------------------------------- +<1> Set the maximum number of lines to sample (the entire sample will be + used if it contains fewer lines) +<2> Request that an explanation of the analysis be returned in the response + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Find File Structure Response + +A +{response}+ contains information about the file structure, +as well as mappings and an ingest pipeline that could be used +to index the contents into {es}. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> The `FileStructure` object contains the structure information From 1cf9436f7dd4eb58d3be2aa935180bf718531c0e Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 23 Nov 2018 09:26:38 +0100 Subject: [PATCH 15/62] Expose all permits acquisition in IndexShard and TransportReplicationAction (#35540) This pull request exposes two new methods in the IndexShard and TransportReplicationAction classes in order to allow transport replication actions to acquire all index shard operation permits for their execution. It first adds the acquireAllPrimaryOperationPermits() and the acquireAllReplicaOperationsPermits() methods to the IndexShard class which allow to acquire all operations permits on a shard while exposing a Releasable. It also refactors the TransportReplicationAction class to expose two protected methods (acquirePrimaryOperationPermit() and acquireReplicaOperationPermit()) that can be overridden when a transport replication action requires the acquisition of all permits on primary and/or replica shard during execution. Finally, it adds a TransportReplicationAllPermitsAcquisitionTests which illustrates how a transport replication action can grab all permits before adding a cluster block in the cluster state, making subsequent operations that requires a single permit to fail). Related to elastic #33888 --- .../TransportReplicationAction.java | 88 +-- .../elasticsearch/index/shard/IndexShard.java | 130 ++-- .../TransportReplicationActionTests.java | 6 +- ...ReplicationAllPermitsAcquisitionTests.java | 561 ++++++++++++++++++ .../index/shard/IndexShardTests.java | 235 ++++++-- 5 files changed, 874 insertions(+), 146 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 38972a7f77462..2938e5edb950b 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -313,7 +313,7 @@ public void messageReceived(ConcreteShardRequest request, TransportChan } } - class AsyncPrimaryAction extends AbstractRunnable implements ActionListener { + class AsyncPrimaryAction extends AbstractRunnable { private final Request request; // targetAllocationID of the shard this request is meant for @@ -334,11 +334,33 @@ class AsyncPrimaryAction extends AbstractRunnable implements ActionListener runWithPrimaryShardReference(new PrimaryShardReference(indexShard, releasable)), + this::onFailure + )); } - @Override - public void onResponse(PrimaryShardReference primaryShardReference) { + void runWithPrimaryShardReference(final PrimaryShardReference primaryShardReference) { try { final ClusterState clusterState = clusterService.state(); final IndexMetaData indexMetaData = clusterState.metaData().getIndexSafe(primaryShardReference.routingEntry().index()); @@ -660,10 +682,10 @@ protected void doRun() throws Exception { setPhase(task, "replica"); final String actualAllocationId = this.replica.routingEntry().allocationId().getId(); if (actualAllocationId.equals(targetAllocationID) == false) { - throw new ShardNotFoundException(this.replica.shardId(), "expected aID [{}] but found [{}]", targetAllocationID, + throw new ShardNotFoundException(this.replica.shardId(), "expected allocation id [{}] but found [{}]", targetAllocationID, actualAllocationId); } - replica.acquireReplicaOperationPermit(primaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, this, executor, request); + acquireReplicaOperationPermit(replica, request, this, primaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes); } /** @@ -697,7 +719,7 @@ public void onFailure(Exception e) { } } - protected IndexShard getIndexShard(ShardId shardId) { + protected IndexShard getIndexShard(final ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); return indexService.getShard(shardId.id()); } @@ -938,42 +960,26 @@ void retryBecauseUnavailable(ShardId shardId, String message) { } /** - * Tries to acquire reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally - * and replication of the operation to all replica shards is completed / failed (see {@link ReplicationOperation}). + * Executes the logic for acquiring one or more operation permit on a primary shard. The default is to acquire a single permit but this + * method can be overridden to acquire more. */ - private void acquirePrimaryShardReference(ShardId shardId, String allocationId, long primaryTerm, - ActionListener onReferenceAcquired, Object debugInfo) { - IndexShard indexShard = getIndexShard(shardId); - // we may end up here if the cluster state used to route the primary is so stale that the underlying - // index shard was replaced with a replica. For example - in a two node cluster, if the primary fails - // the replica will take over and a replica will be assigned to the first node. - if (indexShard.routingEntry().primary() == false) { - throw new ReplicationOperation.RetryOnPrimaryException(indexShard.shardId(), - "actual shard is not a primary " + indexShard.routingEntry()); - } - final String actualAllocationId = indexShard.routingEntry().allocationId().getId(); - if (actualAllocationId.equals(allocationId) == false) { - throw new ShardNotFoundException(shardId, "expected aID [{}] but found [{}]", allocationId, actualAllocationId); - } - final long actualTerm = indexShard.getPendingPrimaryTerm(); - if (actualTerm != primaryTerm) { - throw new ShardNotFoundException(shardId, "expected aID [{}] with term [{}] but found [{}]", allocationId, - primaryTerm, actualTerm); - } - - ActionListener onAcquired = new ActionListener() { - @Override - public void onResponse(Releasable releasable) { - onReferenceAcquired.onResponse(new PrimaryShardReference(indexShard, releasable)); - } - - @Override - public void onFailure(Exception e) { - onReferenceAcquired.onFailure(e); - } - }; + protected void acquirePrimaryOperationPermit(final IndexShard primary, + final Request request, + final ActionListener onAcquired) { + primary.acquirePrimaryOperationPermit(onAcquired, executor, request); + } - indexShard.acquirePrimaryOperationPermit(onAcquired, executor, debugInfo); + /** + * Executes the logic for acquiring one or more operation permit on a replica shard. The default is to acquire a single permit but this + * method can be overridden to acquire more. + */ + protected void acquireReplicaOperationPermit(final IndexShard replica, + final ReplicaRequest request, + final ActionListener onAcquired, + final long primaryTerm, + final long globalCheckpoint, + final long maxSeqNoOfUpdatesOrDeletes) { + replica.acquireReplicaOperationPermit(primaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, onAcquired, executor, request); } class ShardReference implements Releasable { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 0638ce32a147c..05ead45cd128d 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2302,7 +2302,18 @@ public void acquirePrimaryOperationPermit(ActionListener onPermitAcq indexShardOperationPermits.acquire(onPermitAcquired, executorOnDelay, false, debugInfo); } - private void bumpPrimaryTerm(long newPrimaryTerm, final CheckedRunnable onBlocked) { + /** + * Acquire all primary operation permits. Once all permits are acquired, the provided ActionListener is called. + * It is the responsibility of the caller to close the {@link Releasable}. + */ + public void acquireAllPrimaryOperationsPermits(final ActionListener onPermitAcquired, final TimeValue timeout) { + verifyNotClosed(); + assert shardRouting.primary() : "acquireAllPrimaryOperationsPermits should only be called on primary shard: " + shardRouting; + + indexShardOperationPermits.asyncBlockOperations(onPermitAcquired, timeout.duration(), timeout.timeUnit()); + } + + private void bumpPrimaryTerm(final long newPrimaryTerm, final CheckedRunnable onBlocked) { assert Thread.holdsLock(mutex); assert newPrimaryTerm > pendingPrimaryTerm; assert operationPrimaryTerm <= pendingPrimaryTerm; @@ -2357,11 +2368,42 @@ public void onResponse(final Releasable releasable) { public void acquireReplicaOperationPermit(final long opPrimaryTerm, final long globalCheckpoint, final long maxSeqNoOfUpdatesOrDeletes, final ActionListener onPermitAcquired, final String executorOnDelay, final Object debugInfo) { + innerAcquireReplicaOperationPermit(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, onPermitAcquired, + (listener) -> indexShardOperationPermits.acquire(listener, executorOnDelay, true, debugInfo)); + } + + /** + * Acquire all replica operation permits whenever the shard is ready for indexing (see + * {@link #acquireAllPrimaryOperationsPermits(ActionListener, TimeValue)}. If the given primary term is lower than then one in + * {@link #shardRouting}, the {@link ActionListener#onFailure(Exception)} method of the provided listener is invoked with an + * {@link IllegalStateException}. + * + * @param opPrimaryTerm the operation primary term + * @param globalCheckpoint the global checkpoint associated with the request + * @param maxSeqNoOfUpdatesOrDeletes the max seq_no of updates (index operations overwrite Lucene) or deletes captured on the primary + * after this replication request was executed on it (see {@link #getMaxSeqNoOfUpdatesOrDeletes()} + * @param onPermitAcquired the listener for permit acquisition + * @param timeout the maximum time to wait for the in-flight operations block + */ + public void acquireAllReplicaOperationsPermits(final long opPrimaryTerm, + final long globalCheckpoint, + final long maxSeqNoOfUpdatesOrDeletes, + final ActionListener onPermitAcquired, + final TimeValue timeout) { + innerAcquireReplicaOperationPermit(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, onPermitAcquired, + (listener) -> indexShardOperationPermits.asyncBlockOperations(listener, timeout.duration(), timeout.timeUnit())); + } + + private void innerAcquireReplicaOperationPermit(final long opPrimaryTerm, + final long globalCheckpoint, + final long maxSeqNoOfUpdatesOrDeletes, + final ActionListener onPermitAcquired, + final Consumer> consumer) { verifyNotClosed(); if (opPrimaryTerm > pendingPrimaryTerm) { synchronized (mutex) { if (opPrimaryTerm > pendingPrimaryTerm) { - IndexShardState shardState = state(); + final IndexShardState shardState = state(); // only roll translog and update primary term if shard has made it past recovery // Having a new primary term here means that the old primary failed and that there is a new primary, which again // means that the master will fail this shard as all initializing shards are failed when a primary is selected @@ -2373,58 +2415,54 @@ public void acquireReplicaOperationPermit(final long opPrimaryTerm, final long g if (opPrimaryTerm > pendingPrimaryTerm) { bumpPrimaryTerm(opPrimaryTerm, () -> { - updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition"); - final long currentGlobalCheckpoint = getGlobalCheckpoint(); - final long maxSeqNo = seqNoStats().getMaxSeqNo(); - logger.info("detected new primary with primary term [{}], global checkpoint [{}], max_seq_no [{}]", - opPrimaryTerm, currentGlobalCheckpoint, maxSeqNo); - if (currentGlobalCheckpoint < maxSeqNo) { - resetEngineToGlobalCheckpoint(); - } else { - getEngine().rollTranslogGeneration(); - } + updateGlobalCheckpointOnReplica(globalCheckpoint, "primary term transition"); + final long currentGlobalCheckpoint = getGlobalCheckpoint(); + final long maxSeqNo = seqNoStats().getMaxSeqNo(); + logger.info("detected new primary with primary term [{}], global checkpoint [{}], max_seq_no [{}]", + opPrimaryTerm, currentGlobalCheckpoint, maxSeqNo); + if (currentGlobalCheckpoint < maxSeqNo) { + resetEngineToGlobalCheckpoint(); + } else { + getEngine().rollTranslogGeneration(); + } }); } } } } - assert opPrimaryTerm <= pendingPrimaryTerm - : "operation primary term [" + opPrimaryTerm + "] should be at most [" + pendingPrimaryTerm + "]"; - indexShardOperationPermits.acquire( - new ActionListener() { - @Override - public void onResponse(final Releasable releasable) { - if (opPrimaryTerm < operationPrimaryTerm) { - releasable.close(); - final String message = String.format( - Locale.ROOT, - "%s operation primary term [%d] is too old (current [%d])", - shardId, - opPrimaryTerm, - operationPrimaryTerm); - onPermitAcquired.onFailure(new IllegalStateException(message)); - } else { - assert assertReplicationTarget(); - try { - updateGlobalCheckpointOnReplica(globalCheckpoint, "operation"); - advanceMaxSeqNoOfUpdatesOrDeletes(maxSeqNoOfUpdatesOrDeletes); - } catch (Exception e) { - releasable.close(); - onPermitAcquired.onFailure(e); - return; - } - onPermitAcquired.onResponse(releasable); - } - } - - @Override - public void onFailure(final Exception e) { + : "operation primary term [" + opPrimaryTerm + "] should be at most [" + pendingPrimaryTerm + "]"; + consumer.accept(new ActionListener() { + @Override + public void onResponse(final Releasable releasable) { + if (opPrimaryTerm < operationPrimaryTerm) { + releasable.close(); + final String message = String.format( + Locale.ROOT, + "%s operation primary term [%d] is too old (current [%d])", + shardId, + opPrimaryTerm, + operationPrimaryTerm); + onPermitAcquired.onFailure(new IllegalStateException(message)); + } else { + assert assertReplicationTarget(); + try { + updateGlobalCheckpointOnReplica(globalCheckpoint, "operation"); + advanceMaxSeqNoOfUpdatesOrDeletes(maxSeqNoOfUpdatesOrDeletes); + } catch (Exception e) { + releasable.close(); onPermitAcquired.onFailure(e); + return; } - }, - executorOnDelay, - true, debugInfo); + onPermitAcquired.onResponse(releasable); + } + } + + @Override + public void onFailure(final Exception e) { + onPermitAcquired.onFailure(e); + } + }); } public int getActiveOperationsCount() { diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index c8c40a7f5841a..c1991a8f3a17a 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -949,11 +949,11 @@ action.new PrimaryOperationTransportHandler().messageReceived( logger.debug("got exception:" , throwable); assertTrue(throwable.getClass() + " is not a retry exception", action.retryPrimaryException(throwable)); if (wrongAllocationId) { - assertThat(throwable.getMessage(), containsString("expected aID [_not_a_valid_aid_] but found [" + + assertThat(throwable.getMessage(), containsString("expected allocation id [_not_a_valid_aid_] but found [" + primary.allocationId().getId() + "]")); } else { - assertThat(throwable.getMessage(), containsString("expected aID [" + primary.allocationId().getId() + "] with term [" + - requestTerm + "] but found [" + primaryTerm + "]")); + assertThat(throwable.getMessage(), containsString("expected allocation id [" + primary.allocationId().getId() + + "] with term [" + requestTerm + "] but found [" + primaryTerm + "]")); } } } diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java new file mode 100644 index 0000000000000..8cad76bcdfe5e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -0,0 +1,561 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.support.replication; + +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.block.ClusterBlock; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_UUID; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; +import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting; +import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; +import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + + +/** + * This test tests the concurrent execution of several transport replication actions. All of these actions (except one) acquire a single + * permit during their execution on shards and are expected to fail if a global level or index level block is present in the cluster state. + * These actions are all started at the same time, but some are delayed until one last action. + * + * This last action is special because it acquires all the permits on shards, adds the block to the cluster state and then "releases" the + * previously delayed single permit actions. This way, there is a clear transition between the single permit actions executed before the + * all permit action that sets the block and those executed afterwards that are doomed to fail because of the block. + */ +public class TransportReplicationAllPermitsAcquisitionTests extends IndexShardTestCase { + + private ClusterService clusterService; + private TransportService transportService; + private ShardStateAction shardStateAction; + private ShardId shardId; + private IndexShard primary; + private IndexShard replica; + private boolean globalBlock; + private ClusterBlock block; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + globalBlock = randomBoolean(); + RestStatus restStatus = randomFrom(RestStatus.values()); + block = new ClusterBlock(randomIntBetween(1, 10), randomAlphaOfLength(5), false, true, false, restStatus, ClusterBlockLevel.ALL); + clusterService = createClusterService(threadPool); + + final ClusterState.Builder state = ClusterState.builder(clusterService.state()); + Set roles = new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())); + DiscoveryNode node1 = new DiscoveryNode("_name1", "_node1", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); + DiscoveryNode node2 = new DiscoveryNode("_name2", "_node2", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); + state.nodes(DiscoveryNodes.builder() + .add(node1) + .add(node2) + .localNodeId(node1.getId()) + .masterNodeId(node1.getId())); + + shardId = new ShardId("index", UUID.randomUUID().toString(), 0); + ShardRouting shardRouting = + newShardRouting(shardId, node1.getId(), true, ShardRoutingState.INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE); + + Settings indexSettings = Settings.builder() + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .put(SETTING_INDEX_UUID, shardId.getIndex().getUUID()) + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_CREATION_DATE, System.currentTimeMillis()) + .build(); + + primary = newStartedShard(p -> newShard(shardRouting, indexSettings, new InternalEngineFactory()), true); + for (int i = 0; i < 10; i++) { + final String id = Integer.toString(i); + indexDoc(primary, "_doc", id, "{\"value\":" + id + "}"); + } + + IndexMetaData indexMetaData = IndexMetaData.builder(shardId.getIndexName()) + .settings(indexSettings) + .primaryTerm(shardId.id(), primary.getOperationPrimaryTerm()) + .putMapping("_doc","{ \"properties\": { \"value\": { \"type\": \"short\"}}}") + .build(); + state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded()); + + replica = newShard(primary.shardId(), false, node2.getId(), indexMetaData, null); + recoverReplica(replica, primary, true); + + IndexRoutingTable.Builder routing = IndexRoutingTable.builder(indexMetaData.getIndex()); + routing.addIndexShard(new IndexShardRoutingTable.Builder(shardId) + .addShard(primary.routingEntry()) + .build()); + state.routingTable(RoutingTable.builder().add(routing.build()).build()); + + setState(clusterService, state.build()); + + final Settings transportSettings = Settings.builder().put("node.name", node1.getId()).build(); + transportService = MockTransportService.createNewService(transportSettings, Version.CURRENT, threadPool, null); + transportService.start(); + transportService.acceptIncomingRequests(); + shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool); + } + + @Override + @After + public void tearDown() throws Exception { + closeShards(primary, replica); + transportService.stop(); + clusterService.close(); + super.tearDown(); + } + + public void testTransportReplicationActionWithAllPermits() throws Exception { + final int numOperations = scaledRandomIntBetween(4, 32); + final int delayedOperations = randomIntBetween(1, numOperations); + logger.trace("starting [{}] operations, among which the first [{}] started ops should be blocked by [{}]", + numOperations, delayedOperations, block); + + final CyclicBarrier delayedOperationsBarrier = new CyclicBarrier(delayedOperations + 1); + final List threads = new ArrayList<>(delayedOperationsBarrier.getParties()); + + @SuppressWarnings("unchecked") + final PlainActionFuture[] futures = new PlainActionFuture[numOperations]; + final TestAction[] actions = new TestAction[numOperations]; + + for (int i = 0; i < numOperations; i++) { + final int threadId = i; + final boolean delayed = (threadId < delayedOperations); + + final PlainActionFuture listener = new PlainActionFuture<>(); + futures[threadId] = listener; + + final TestAction singlePermitAction = new SinglePermitWithBlocksAction(Settings.EMPTY, "internalSinglePermit[" + threadId + "]", + transportService, clusterService, shardStateAction, threadPool, shardId, primary, replica, globalBlock); + actions[threadId] = singlePermitAction; + + Thread thread = new Thread(() -> { + TransportReplicationAction.AsyncPrimaryAction asyncPrimaryAction = + singlePermitAction.new AsyncPrimaryAction(request(), allocationId(), primaryTerm(), transportChannel(listener), null) { + @Override + protected void doRun() throws Exception { + if (delayed) { + logger.trace("op [{}] has started and will resume execution once allPermitsAction is terminated", threadId); + delayedOperationsBarrier.await(); + } + super.doRun(); + } + + @Override + void runWithPrimaryShardReference(final TransportReplicationAction.PrimaryShardReference reference) { + assertThat(reference.indexShard.getActiveOperationsCount(), greaterThan(0)); + assertSame(primary, reference.indexShard); + assertBlockIsPresentForDelayedOp(); + super.runWithPrimaryShardReference(reference); + } + + @Override + public void onFailure(Exception e) { + assertBlockIsPresentForDelayedOp(); + super.onFailure(e); + } + + private void assertBlockIsPresentForDelayedOp() { + if (delayed) { + final ClusterState clusterState = clusterService.state(); + if (globalBlock) { + assertTrue("Global block must exist", clusterState.blocks().hasGlobalBlock(block)); + } else { + String indexName = primary.shardId().getIndexName(); + assertTrue("Index block must exist", clusterState.blocks().hasIndexBlock(indexName, block)); + } + } + } + }; + asyncPrimaryAction.run(); + }); + threads.add(thread); + thread.start(); + } + + logger.trace("now starting the operation that acquires all permits and sets the block in the cluster state"); + + // An action which acquires all operation permits during execution and set a block + final TestAction allPermitsAction = new AllPermitsThenBlockAction(Settings.EMPTY, "internalAllPermits", transportService, + clusterService, shardStateAction, threadPool, shardId, primary, replica); + + final PlainActionFuture allPermitFuture = new PlainActionFuture<>(); + Thread thread = new Thread(() -> { + TransportReplicationAction.AsyncPrimaryAction asyncPrimaryAction = + allPermitsAction.new AsyncPrimaryAction(request(), allocationId(), primaryTerm(), transportChannel(allPermitFuture), null) { + @Override + void runWithPrimaryShardReference(final TransportReplicationAction.PrimaryShardReference reference) { + assertEquals("All permits must be acquired", 0, reference.indexShard.getActiveOperationsCount()); + assertSame(primary, reference.indexShard); + + final ClusterState clusterState = clusterService.state(); + final ClusterBlocks.Builder blocks = ClusterBlocks.builder(); + if (globalBlock) { + assertFalse("Global block must not exist yet", clusterState.blocks().hasGlobalBlock(block)); + blocks.addGlobalBlock(block); + } else { + String indexName = reference.indexShard.shardId().getIndexName(); + assertFalse("Index block must not exist yet", clusterState.blocks().hasIndexBlock(indexName, block)); + blocks.addIndexBlock(indexName, block); + } + + logger.trace("adding test block to cluster state {}", block); + setState(clusterService, ClusterState.builder(clusterState).blocks(blocks)); + + try { + logger.trace("releasing delayed operations"); + delayedOperationsBarrier.await(); + } catch (InterruptedException | BrokenBarrierException e) { + onFailure(e); + } + super.runWithPrimaryShardReference(reference); + } + }; + asyncPrimaryAction.run(); + }); + threads.add(thread); + thread.start(); + + logger.trace("waiting for all operations to terminate"); + for (Thread t : threads) { + t.join(); + } + + final Response allPermitsResponse = allPermitFuture.get(); + assertSuccessfulOperation(allPermitsAction, allPermitsResponse); + + for (int i = 0; i < numOperations; i++) { + final PlainActionFuture future = futures[i]; + final TestAction action = actions[i]; + + if (i < delayedOperations) { + ExecutionException exception = expectThrows(ExecutionException.class, "delayed operation should have failed", future::get); + assertFailedOperation(action, exception); + } else { + // non delayed operation might fail depending on the order they were executed + try { + assertSuccessfulOperation(action, futures[i].get()); + } catch (final ExecutionException e) { + assertFailedOperation(action, e); + } + } + } + } + + private void assertSuccessfulOperation(final TestAction action, final Response response) { + final String name = action.getActionName(); + assertThat(name + " operation should have been executed on primary", action.executedOnPrimary.get(), is(true)); + assertThat(name + " operation should have been executed on replica", action.executedOnReplica.get(), is(true)); + assertThat(name + " operation must have a non null result", response, notNullValue()); + assertThat(name + " operation should have been successful on 2 shards", response.getShardInfo().getSuccessful(), equalTo(2)); + } + + private void assertFailedOperation(final TestAction action,final ExecutionException exception) { + final String name = action.getActionName(); + assertThat(name + " operation should not have been executed on primary", action.executedOnPrimary.get(), nullValue()); + assertThat(name + " operation should not have been executed on replica", action.executedOnReplica.get(), nullValue()); + assertThat(exception.getCause(), instanceOf(ClusterBlockException.class)); + ClusterBlockException clusterBlockException = (ClusterBlockException) exception.getCause(); + assertThat(clusterBlockException.blocks(), hasItem(equalTo(block))); + } + + private long primaryTerm() { + return primary.getOperationPrimaryTerm(); + } + + private String allocationId() { + return primary.routingEntry().allocationId().getId(); + } + + private Request request() { + return new Request().setShardId(primary.shardId()); + } + + /** + * A type of {@link TransportReplicationAction} that allows to use the primary and replica shards passed to the constructor for the + * execution of the replication action. Also records if the operation is executed on the primary and the replica. + */ + private abstract class TestAction extends TransportReplicationAction { + + protected final ShardId shardId; + protected final IndexShard primary; + protected final IndexShard replica; + protected final SetOnce executedOnPrimary = new SetOnce<>(); + protected final SetOnce executedOnReplica = new SetOnce<>(); + + TestAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, + ShardStateAction shardStateAction, ThreadPool threadPool, ShardId shardId, IndexShard primary, IndexShard replica) { + super(settings, actionName, transportService, clusterService, null, threadPool, shardStateAction, + new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(), Request::new, Request::new, ThreadPool.Names.SAME); + this.shardId = Objects.requireNonNull(shardId); + this.primary = Objects.requireNonNull(primary); + assertEquals(shardId, primary.shardId()); + this.replica = Objects.requireNonNull(replica); + assertEquals(shardId, replica.shardId()); + } + + @Override + protected Response newResponseInstance() { + return new Response(); + } + + public String getActionName() { + return this.actionName; + } + + @Override + protected PrimaryResult shardOperationOnPrimary(Request shardRequest, IndexShard shard) throws Exception { + executedOnPrimary.set(true); + // The TransportReplicationAction.getIndexShard() method is overridden for testing purpose but we double check here + // that the permit has been acquired on the primary shard + assertSame(primary, shard); + return new PrimaryResult<>(shardRequest, new Response()); + } + + @Override + protected ReplicaResult shardOperationOnReplica(Request shardRequest, IndexShard shard) throws Exception { + executedOnReplica.set(true); + // The TransportReplicationAction.getIndexShard() method is overridden for testing purpose but we double check here + // that the permit has been acquired on the replica shard + assertSame(replica, shard); + return new ReplicaResult(); + } + + @Override + protected IndexShard getIndexShard(final ShardId shardId) { + if (this.shardId.equals(shardId) == false) { + throw new AssertionError("shard id differs from " + shardId); + } + return (executedOnPrimary.get() == null) ? primary : replica; + } + + @Override + protected void sendReplicaRequest(final ConcreteReplicaRequest replicaRequest, + final DiscoveryNode node, + final ActionListener listener) { + assertEquals("Replica is always assigned to node 2 in this test", clusterService.state().nodes().get("_node2"), node); + ReplicaOperationTransportHandler replicaOperationTransportHandler = new ReplicaOperationTransportHandler(); + try { + replicaOperationTransportHandler.messageReceived(replicaRequest, new TransportChannel() { + @Override + public String getProfileName() { + return null; + } + + @Override + public String getChannelType() { + return null; + } + + @Override + public void sendResponse(TransportResponse response) throws IOException { + listener.onResponse((ReplicationOperation.ReplicaResponse) response); + } + + @Override + public void sendResponse(Exception exception) throws IOException { + listener.onFailure(exception); + } + }, null); + } catch (Exception e) { + listener.onFailure(e); + } + } + } + + /** + * A type of {@link TransportReplicationAction} that acquires a single permit during execution and that blocks + * on {@link ClusterBlockLevel#WRITE}. The block can be a global level or an index level block depending of the + * value of the {@code globalBlock} parameter in the constructor. When the operation is executed on shards it + * verifies that at least 1 permit is acquired and that there is no blocks in the cluster state. + */ + private class SinglePermitWithBlocksAction extends TestAction { + + private final boolean globalBlock; + + SinglePermitWithBlocksAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, + ShardStateAction shardStateAction, ThreadPool threadPool, + ShardId shardId, IndexShard primary, IndexShard replica, boolean globalBlock) { + super(settings, actionName, transportService, clusterService, shardStateAction, threadPool, shardId, primary, replica); + this.globalBlock = globalBlock; + } + + @Override + protected ClusterBlockLevel globalBlockLevel() { + return globalBlock ? ClusterBlockLevel.WRITE : super.globalBlockLevel(); + } + + @Override + protected ClusterBlockLevel indexBlockLevel() { + return globalBlock == false ? ClusterBlockLevel.WRITE : super.indexBlockLevel(); + } + + @Override + protected PrimaryResult shardOperationOnPrimary(Request shardRequest, IndexShard shard) throws Exception { + assertNoBlocks("block must not exist when executing the operation on primary shard: it should have been blocked before"); + assertThat(shard.getActiveOperationsCount(), greaterThan(0)); + return super.shardOperationOnPrimary(shardRequest, shard); + } + + @Override + protected ReplicaResult shardOperationOnReplica(Request shardRequest, IndexShard shard) throws Exception { + assertNoBlocks("block must not exist when executing the operation on replica shard: it should have been blocked before"); + assertThat(shard.getActiveOperationsCount(), greaterThan(0)); + return super.shardOperationOnReplica(shardRequest, shard); + } + + private void assertNoBlocks(final String error) { + final ClusterState clusterState = clusterService.state(); + assertFalse("Global level " + error, clusterState.blocks().hasGlobalBlock(block)); + assertFalse("Index level " + error, clusterState.blocks().hasIndexBlock(shardId.getIndexName(), block)); + } + } + + /** + * A type of {@link TransportReplicationAction} that acquires all permits during execution. + */ + private class AllPermitsThenBlockAction extends TestAction { + + private final TimeValue timeout = TimeValue.timeValueSeconds(30L); + + AllPermitsThenBlockAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, + ShardStateAction shardStateAction, ThreadPool threadPool, + ShardId shardId, IndexShard primary, IndexShard replica) { + super(settings, actionName, transportService, clusterService, shardStateAction, threadPool, shardId, primary, replica); + } + + @Override + protected void acquirePrimaryOperationPermit(IndexShard shard, Request request, ActionListener onAcquired) { + shard.acquireAllPrimaryOperationsPermits(onAcquired, timeout); + } + + @Override + protected void acquireReplicaOperationPermit(IndexShard shard, Request request, ActionListener onAcquired, + long primaryTerm, long globalCheckpoint, long maxSeqNo) { + shard.acquireAllReplicaOperationsPermits(primaryTerm, globalCheckpoint, maxSeqNo, onAcquired, timeout); + } + + @Override + protected PrimaryResult shardOperationOnPrimary(Request shardRequest, IndexShard shard) throws Exception { + assertEquals("All permits must be acquired", 0, shard.getActiveOperationsCount()); + return super.shardOperationOnPrimary(shardRequest, shard); + } + + @Override + protected ReplicaResult shardOperationOnReplica(Request shardRequest, IndexShard shard) throws Exception { + assertEquals("All permits must be acquired", 0, shard.getActiveOperationsCount()); + return super.shardOperationOnReplica(shardRequest, shard); + } + } + + static class Request extends ReplicationRequest { + @Override + public String toString() { + return getTestClass().getName() + ".Request"; + } + } + + static class Response extends ReplicationResponse { + } + + /** + * Transport channel that is needed for replica operation testing. + */ + public TransportChannel transportChannel(final PlainActionFuture listener) { + return new TransportChannel() { + + @Override + public String getProfileName() { + return ""; + } + + @Override + public void sendResponse(TransportResponse response) throws IOException { + listener.onResponse(((Response) response)); + } + + @Override + public void sendResponse(Exception exception) throws IOException { + listener.onFailure(exception); + } + + @Override + public String getChannelType() { + return "replica_test"; + } + }; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 8eede7542bd75..1baa61e144b73 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -34,6 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; import org.elasticsearch.Assertions; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; @@ -60,6 +61,7 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -69,6 +71,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -124,7 +127,6 @@ import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.ElasticsearchException; import java.io.IOException; import java.nio.charset.Charset; @@ -304,30 +306,27 @@ public void testShardStateMetaHashCodeEquals() { } - public void testClosesPreventsNewOperations() throws InterruptedException, ExecutionException, IOException { + public void testClosesPreventsNewOperations() throws Exception { IndexShard indexShard = newStartedShard(); closeShards(indexShard); assertThat(indexShard.getActiveOperationsCount(), equalTo(0)); - try { - indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE, ""); - fail("we should not be able to increment anymore"); - } catch (IndexShardClosedException e) { - // expected - } - try { - indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, - randomNonNegativeLong(), null, ThreadPool.Names.WRITE, ""); - fail("we should not be able to increment anymore"); - } catch (IndexShardClosedException e) { - // expected - } + expectThrows(IndexShardClosedException.class, + () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE, "")); + expectThrows(IndexShardClosedException.class, + () -> indexShard.acquireAllPrimaryOperationsPermits(null, TimeValue.timeValueSeconds(30L))); + expectThrows(IndexShardClosedException.class, + () -> indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), null, ThreadPool.Names.WRITE, "")); + expectThrows(IndexShardClosedException.class, + () -> indexShard.acquireAllReplicaOperationsPermits(indexShard.getPendingPrimaryTerm(), SequenceNumbers.UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), null, TimeValue.timeValueSeconds(30L))); } public void testRejectOperationPermitWithHigherTermWhenNotStarted() throws IOException { IndexShard indexShard = newShard(false); expectThrows(IndexShardNotStartedException.class, () -> - indexShard.acquireReplicaOperationPermit(indexShard.getPendingPrimaryTerm() + randomIntBetween(1, 100), - SequenceNumbers.UNASSIGNED_SEQ_NO, randomNonNegativeLong(), null, ThreadPool.Names.WRITE, "")); + randomReplicaOperationPermitAcquisition(indexShard, indexShard.getPendingPrimaryTerm() + randomIntBetween(1, 100), + SequenceNumbers.UNASSIGNED_SEQ_NO, randomNonNegativeLong(), null, "")); closeShards(indexShard); } @@ -620,6 +619,106 @@ public void onFailure(Exception e) { closeShards(indexShard); } + public void testAcquirePrimaryAllOperationsPermits() throws Exception { + final IndexShard indexShard = newStartedShard(true); + assertEquals(0, indexShard.getActiveOperationsCount()); + + final CountDownLatch allPermitsAcquired = new CountDownLatch(1); + + final Thread[] threads = new Thread[randomIntBetween(2, 5)]; + final List> futures = new ArrayList<>(threads.length); + final AtomicArray> results = new AtomicArray<>(threads.length); + final CountDownLatch allOperationsDone = new CountDownLatch(threads.length); + + for (int i = 0; i < threads.length; i++) { + final int threadId = i; + final boolean singlePermit = randomBoolean(); + + final PlainActionFuture future = new PlainActionFuture() { + @Override + public void onResponse(final Releasable releasable) { + if (singlePermit) { + assertThat(indexShard.getActiveOperationsCount(), greaterThan(0)); + } else { + assertThat(indexShard.getActiveOperationsCount(), equalTo(0)); + } + releasable.close(); + super.onResponse(releasable); + results.setOnce(threadId, Tuple.tuple(Boolean.TRUE, null)); + allOperationsDone.countDown(); + } + + @Override + public void onFailure(final Exception e) { + results.setOnce(threadId, Tuple.tuple(Boolean.FALSE, e)); + allOperationsDone.countDown(); + } + }; + futures.add(threadId, future); + + threads[threadId] = new Thread(() -> { + try { + allPermitsAcquired.await(); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + if (singlePermit) { + indexShard.acquirePrimaryOperationPermit(future, ThreadPool.Names.WRITE, ""); + } else { + indexShard.acquireAllPrimaryOperationsPermits(future, TimeValue.timeValueHours(1L)); + } + assertEquals(0, indexShard.getActiveOperationsCount()); + }); + threads[threadId].start(); + } + + final AtomicBoolean blocked = new AtomicBoolean(); + final CountDownLatch allPermitsTerminated = new CountDownLatch(1); + + final PlainActionFuture futureAllPermits = new PlainActionFuture() { + @Override + public void onResponse(final Releasable releasable) { + try { + blocked.set(true); + allPermitsAcquired.countDown(); + super.onResponse(releasable); + allPermitsTerminated.await(); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + indexShard.acquireAllPrimaryOperationsPermits(futureAllPermits, TimeValue.timeValueSeconds(30L)); + allPermitsAcquired.await(); + assertTrue(blocked.get()); + assertEquals(0, indexShard.getActiveOperationsCount()); + assertTrue("Expected no results, operations are blocked", results.asList().isEmpty()); + futures.forEach(future -> assertFalse(future.isDone())); + + allPermitsTerminated.countDown(); + + final Releasable allPermits = futureAllPermits.get(); + assertTrue(futureAllPermits.isDone()); + + assertTrue("Expected no results, operations are blocked", results.asList().isEmpty()); + futures.forEach(future -> assertFalse(future.isDone())); + + Releasables.close(allPermits); + allOperationsDone.await(); + for (Thread thread : threads) { + thread.join(); + } + + futures.forEach(future -> assertTrue(future.isDone())); + assertEquals(threads.length, results.asList().size()); + results.asList().forEach(result -> { + assertTrue(result.v1()); + assertNull(result.v2()); + }); + + closeShards(indexShard); + } + private Releasable acquirePrimaryOperationPermitBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException { PlainActionFuture fut = new PlainActionFuture<>(); indexShard.acquirePrimaryOperationPermit(fut, ThreadPool.Names.WRITE, ""); @@ -676,10 +775,14 @@ public void testOperationPermitOnReplicaShards() throws Exception { assertEquals(0, indexShard.getActiveOperationsCount()); if (shardRouting.primary() == false && Assertions.ENABLED) { - final AssertionError e = + AssertionError e = expectThrows(AssertionError.class, () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE, "")); assertThat(e, hasToString(containsString("acquirePrimaryOperationPermit should only be called on primary shard"))); + + e = expectThrows(AssertionError.class, + () -> indexShard.acquireAllPrimaryOperationsPermits(null, TimeValue.timeValueSeconds(30L))); + assertThat(e, hasToString(containsString("acquireAllPrimaryOperationsPermits should only be called on primary shard"))); } final long primaryTerm = indexShard.getPendingPrimaryTerm(); @@ -697,34 +800,6 @@ public void testOperationPermitOnReplicaShards() throws Exception { operation2 = null; } - { - final AtomicBoolean onResponse = new AtomicBoolean(); - final AtomicBoolean onFailure = new AtomicBoolean(); - final AtomicReference onFailureException = new AtomicReference<>(); - ActionListener onLockAcquired = new ActionListener() { - @Override - public void onResponse(Releasable releasable) { - onResponse.set(true); - } - - @Override - public void onFailure(Exception e) { - onFailure.set(true); - onFailureException.set(e); - } - }; - - indexShard.acquireReplicaOperationPermit(primaryTerm - 1, SequenceNumbers.UNASSIGNED_SEQ_NO, - randomNonNegativeLong(), onLockAcquired, ThreadPool.Names.WRITE, ""); - - assertFalse(onResponse.get()); - assertTrue(onFailure.get()); - assertThat(onFailureException.get(), instanceOf(IllegalStateException.class)); - assertThat( - onFailureException.get(), - hasToString(containsString("operation primary term [" + (primaryTerm - 1) + "] is too old"))); - } - { final AtomicBoolean onResponse = new AtomicBoolean(); final AtomicReference onFailure = new AtomicReference<>(); @@ -785,12 +860,12 @@ private void finish() { } }; try { - indexShard.acquireReplicaOperationPermit( + randomReplicaOperationPermitAcquisition(indexShard, newPrimaryTerm, newGlobalCheckPoint, randomNonNegativeLong(), listener, - ThreadPool.Names.SAME, ""); + ""); } catch (Exception e) { listener.onFailure(e); } @@ -837,6 +912,37 @@ private void finish() { assertEquals(0, indexShard.getActiveOperationsCount()); } + { + final CountDownLatch latch = new CountDownLatch(1); + final AtomicBoolean onResponse = new AtomicBoolean(); + final AtomicBoolean onFailure = new AtomicBoolean(); + final AtomicReference onFailureException = new AtomicReference<>(); + ActionListener onLockAcquired = new ActionListener() { + @Override + public void onResponse(Releasable releasable) { + onResponse.set(true); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + onFailure.set(true); + onFailureException.set(e); + latch.countDown(); + } + }; + + final long oldPrimaryTerm = indexShard.pendingPrimaryTerm - 1; + randomReplicaOperationPermitAcquisition(indexShard, oldPrimaryTerm, indexShard.getGlobalCheckpoint(), + randomNonNegativeLong(), onLockAcquired, ""); + latch.await(); + assertFalse(onResponse.get()); + assertTrue(onFailure.get()); + assertThat(onFailureException.get(), instanceOf(IllegalStateException.class)); + assertThat( + onFailureException.get(), hasToString(containsString("operation primary term [" + oldPrimaryTerm + "] is too old"))); + } + closeShards(indexShard); } @@ -848,8 +954,8 @@ public void testAcquireReplicaPermitAdvanceMaxSeqNoOfUpdates() throws Exception long newMaxSeqNoOfUpdates = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); PlainActionFuture fut = new PlainActionFuture<>(); - replica.acquireReplicaOperationPermit(replica.operationPrimaryTerm, replica.getGlobalCheckpoint(), - newMaxSeqNoOfUpdates, fut, ThreadPool.Names.WRITE, ""); + randomReplicaOperationPermitAcquisition(replica, replica.operationPrimaryTerm, replica.getGlobalCheckpoint(), + newMaxSeqNoOfUpdates, fut, ""); try (Releasable ignored = fut.actionGet()) { assertThat(replica.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(Math.max(currentMaxSeqNoOfUpdates, newMaxSeqNoOfUpdates))); } @@ -932,7 +1038,7 @@ public void testRestoreLocalHistoryFromTranslogOnPromotion() throws IOException, final Set docsBeforeRollback = getShardDocUIDs(indexShard); final CountDownLatch latch = new CountDownLatch(1); final boolean shouldRollback = Math.max(globalCheckpointOnReplica, globalCheckpoint) < maxSeqNo; - indexShard.acquireReplicaOperationPermit( + randomReplicaOperationPermitAcquisition(indexShard, indexShard.getPendingPrimaryTerm() + 1, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, @@ -947,8 +1053,7 @@ public void onResponse(Releasable releasable) { public void onFailure(Exception e) { } - }, - ThreadPool.Names.SAME, ""); + }, ""); latch.await(); if (shouldRollback) { @@ -999,7 +1104,7 @@ public void testRollbackReplicaEngineOnPromotion() throws IOException, Interrupt && indexShard.seqNoStats().getMaxSeqNo() != SequenceNumbers.NO_OPS_PERFORMED; final Engine beforeRollbackEngine = indexShard.getEngine(); final long newMaxSeqNoOfUpdates = randomLongBetween(indexShard.getMaxSeqNoOfUpdatesOrDeletes(), Long.MAX_VALUE); - indexShard.acquireReplicaOperationPermit( + randomReplicaOperationPermitAcquisition(indexShard, indexShard.pendingPrimaryTerm + 1, globalCheckpoint, newMaxSeqNoOfUpdates, @@ -1014,8 +1119,7 @@ public void onResponse(final Releasable releasable) { public void onFailure(final Exception e) { } - }, - ThreadPool.Names.SAME, ""); + }, ""); latch.await(); if (globalCheckpointOnReplica == SequenceNumbers.UNASSIGNED_SEQ_NO && globalCheckpoint == SequenceNumbers.UNASSIGNED_SEQ_NO) { @@ -3497,4 +3601,23 @@ public void testResetEngine() throws Exception { public Settings threadPoolSettings() { return Settings.builder().put(super.threadPoolSettings()).put("thread_pool.estimated_time_interval", "5ms").build(); } + + /** + * Randomizes the usage of {@link IndexShard#acquireReplicaOperationPermit(long, long, long, ActionListener, String, Object)} and + * {@link IndexShard#acquireAllReplicaOperationsPermits(long, long, long, ActionListener, TimeValue)} in order to acquire a permit. + */ + private void randomReplicaOperationPermitAcquisition(final IndexShard indexShard, + final long opPrimaryTerm, + final long globalCheckpoint, + final long maxSeqNoOfUpdatesOrDeletes, + final ActionListener listener, + final String info) { + if (randomBoolean()) { + final String executor = ThreadPool.Names.WRITE; + indexShard.acquireReplicaOperationPermit(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener, executor, info); + } else { + final TimeValue timeout = TimeValue.timeValueSeconds(30L); + indexShard.acquireAllReplicaOperationsPermits(opPrimaryTerm, globalCheckpoint, maxSeqNoOfUpdatesOrDeletes, listener, timeout); + } + } } From 9b96fc80d10941f7f24af4e6e7becd2a5bf26054 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 23 Nov 2018 09:42:23 +0100 Subject: [PATCH 16/62] Fix analyzed prefix query in query_string (#35756) This change fixes analyzed prefix queries in `query_string` to be ignored if all terms are removed during the analysis. Closes #31702 --- .../index/search/QueryStringQueryParser.java | 11 +++++++---- .../index/query/QueryStringQueryBuilderTests.java | 10 ++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java index bcf1e92431587..84597d4d3383c 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryStringQueryParser.java @@ -484,10 +484,13 @@ protected Query getPrefixQuery(String field, String termStr) throws ParseExcepti List queries = new ArrayList<>(); for (Map.Entry entry : fields.entrySet()) { Query q = getPrefixQuerySingle(entry.getKey(), termStr); - assert q != null; - queries.add(applyBoost(q, entry.getValue())); + if (q != null) { + queries.add(applyBoost(q, entry.getValue())); + } } - if (queries.size() == 1) { + if (queries.isEmpty()) { + return null; + } else if (queries.size() == 1) { return queries.get(0); } else { float tiebreaker = groupTieBreaker == null ? type.tieBreaker() : groupTieBreaker; @@ -561,7 +564,7 @@ private Query getPossiblyAnalyzedPrefixQuery(String field, String termStr) throw } if (tlist.size() == 0) { - return super.getPrefixQuery(field, termStr); + return null; } if (tlist.size() == 1 && tlist.get(0).size() == 1) { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index bb3f2751fa815..9ea98cebe711b 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -1422,6 +1422,16 @@ public void testPhraseSlop() throws Exception { assertEquals(expected, query); } + public void testAnalyzedPrefix() throws Exception { + Query query = new QueryStringQueryBuilder("quick* @&*") + .field(STRING_FIELD_NAME) + .analyzer("standard") + .analyzeWildcard(true) + .toQuery(createShardContext()); + Query expected = new PrefixQuery(new Term(STRING_FIELD_NAME, "quick")); + assertEquals(expected, query); + } + private static IndexMetaData newIndexMeta(String name, Settings oldIndexSettings, Settings indexSettings) { Settings build = Settings.builder().put(oldIndexSettings) .put(indexSettings) From d3db6c6ecdfbe63f12544a70bbd32cfd9f516cfd Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 23 Nov 2018 08:43:34 +0000 Subject: [PATCH 17/62] Copy checkpoint atomically when rolling generation (#35407) Today when rolling a transog generation we copy the checkpoint from `translog.ckp` to `translog-nnnn.ckp` using a simple `Files.copy()` followed by appropriate `fsync()` calls. The copy operation is not atomic, so if we crash at the wrong moment we can leave an incomplete checkpoint file on disk. In practice the checkpoint is so small that it's either empty or fully written. However, we do not correctly handle the case where it's empty when the node restarts. In contrast, in `recoverFromFiles()` we _do_ copy the checkpoint atomically. This commit extracts the atomic copy operation from `recoverFromFiles()` and re-uses it in `rollGeneration()`. --- .../index/translog/Translog.java | 41 ++++++++++--------- .../index/translog/TranslogTests.java | 34 +++++++++++++++ 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 513d044f735ae..83d81222bf58f 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -213,9 +213,6 @@ public Translog( private ArrayList recoverFromFiles(Checkpoint checkpoint) throws IOException { boolean success = false; ArrayList foundTranslogs = new ArrayList<>(); - // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work - final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, TRANSLOG_FILE_SUFFIX); - boolean tempFileRenamed = false; try (ReleasableLock lock = writeLock.acquire()) { logger.debug("open uncommitted translog checkpoint {}", checkpoint); @@ -263,20 +260,32 @@ private ArrayList recoverFromFiles(Checkpoint checkpoint) throws " already exists but has corrupted content expected: " + checkpoint + " but got: " + checkpointFromDisk); } } else { - // we first copy this into the temp-file and then fsync it followed by an atomic move into the target file - // that way if we hit a disk-full here we are still in an consistent state. - Files.copy(location.resolve(CHECKPOINT_FILE_NAME), tempFile, StandardCopyOption.REPLACE_EXISTING); - IOUtils.fsync(tempFile, false); - Files.move(tempFile, commitCheckpoint, StandardCopyOption.ATOMIC_MOVE); - tempFileRenamed = true; - // we only fsync the directory the tempFile was already fsynced - IOUtils.fsync(commitCheckpoint.getParent(), true); + copyCheckpointTo(commitCheckpoint); } success = true; } finally { if (success == false) { IOUtils.closeWhileHandlingException(foundTranslogs); } + } + return foundTranslogs; + } + + private void copyCheckpointTo(Path targetPath) throws IOException { + // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work + final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, CHECKPOINT_SUFFIX); + boolean tempFileRenamed = false; + + try { + // we first copy this into the temp-file and then fsync it followed by an atomic move into the target file + // that way if we hit a disk-full here we are still in an consistent state. + Files.copy(location.resolve(CHECKPOINT_FILE_NAME), tempFile, StandardCopyOption.REPLACE_EXISTING); + IOUtils.fsync(tempFile, false); + Files.move(tempFile, targetPath, StandardCopyOption.ATOMIC_MOVE); + tempFileRenamed = true; + // we only fsync the directory the tempFile was already fsynced + IOUtils.fsync(targetPath.getParent(), true); + } finally { if (tempFileRenamed == false) { try { Files.delete(tempFile); @@ -285,7 +294,6 @@ private ArrayList recoverFromFiles(Checkpoint checkpoint) throws } } } - return foundTranslogs; } TranslogReader openReader(Path path, Checkpoint checkpoint) throws IOException { @@ -1643,13 +1651,8 @@ public void rollGeneration() throws IOException { try { final TranslogReader reader = current.closeIntoReader(); readers.add(reader); - final Path checkpoint = location.resolve(CHECKPOINT_FILE_NAME); - assert Checkpoint.read(checkpoint).generation == current.getGeneration(); - final Path generationCheckpoint = - location.resolve(getCommitCheckpointFileName(current.getGeneration())); - Files.copy(checkpoint, generationCheckpoint); - IOUtils.fsync(generationCheckpoint, false); - IOUtils.fsync(generationCheckpoint.getParent(), true); + assert Checkpoint.read(location.resolve(CHECKPOINT_FILE_NAME)).generation == current.getGeneration(); + copyCheckpointTo(location.resolve(getCommitCheckpointFileName(current.getGeneration()))); // create a new translog file; this will sync it and update the checkpoint data; current = createWriter(current.getGeneration() + 1); logger.trace("current translog set to [{}]", current.getGeneration()); diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 915cbdd260cef..7ff80328e380f 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.Term; import org.apache.lucene.mockfile.FilterFileChannel; +import org.apache.lucene.mockfile.FilterFileSystemProvider; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.MockDirectoryWrapper; @@ -81,6 +82,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.nio.file.CopyOption; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -3197,4 +3199,36 @@ public void close() throws IOException { snapshot.close(); } } + + public void testCrashDuringCheckpointCopy() throws IOException { + final Path path = createTempDir(); + final AtomicBoolean failOnCopy = new AtomicBoolean(); + final String expectedExceptionMessage = "simulated failure during copy"; + final FilterFileSystemProvider filterFileSystemProvider + = new FilterFileSystemProvider(path.getFileSystem().provider().getScheme(), path.getFileSystem()) { + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + if (failOnCopy.get() && source.toString().endsWith(Translog.CHECKPOINT_SUFFIX)) { + deleteIfExists(target); + Files.createFile(target); + throw new IOException(expectedExceptionMessage); + } else { + super.copy(source, target, options); + } + } + }; + + try (Translog brokenTranslog = create(filterFileSystemProvider.getPath(path.toUri()))) { + failOnCopy.set(true); + assertThat(expectThrows(IOException.class, brokenTranslog::rollGeneration).getMessage(), equalTo(expectedExceptionMessage)); + assertFalse(brokenTranslog.isOpen()); + + try (Translog recoveredTranslog = new Translog(getTranslogConfig(path), brokenTranslog.getTranslogUUID(), + brokenTranslog.getDeletionPolicy(), () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get)) { + recoveredTranslog.rollGeneration(); + assertFilePresences(recoveredTranslog); + } + } + } } From 87454cbf0d626b5e5491593013794b86ea8f3022 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 19 Jan 2019 11:29:17 +0100 Subject: [PATCH 18/62] start --- .../snapshots/SnapshotsServiceTests.java | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 1531744a13cac..36fedd8011b9e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -98,6 +98,7 @@ import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.disruption.DisruptableMockTransport; +import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.junit.After; @@ -129,6 +130,15 @@ public class SnapshotsServiceTests extends ESTestCase { + private static final Logger LOGGER = LogManager.getLogger(SnapshotsServiceTests.class); + + private static final NetworkDisruption.DisruptedLinks NO_DISRUPTION = new NetworkDisruption.DisruptedLinks() { + @Override + public boolean disrupt(final String node1, final String node2) { + return false; + } + }; + private DeterministicTaskQueue deterministicTaskQueue; private TestClusterNodes testClusterNodes; @@ -196,6 +206,48 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } + public void testSnapshotWithMasterFailover() { + setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); + + String repoName = "repo"; + String snapshotName = "snapshot"; + final String index = "test"; + + final int shards = randomIntBetween(1, 10); + + TestClusterNode masterNode = + testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state()); + final AtomicBoolean createdSnapshot = new AtomicBoolean(); + masterNode.client.admin().cluster().preparePutRepository(repoName) + .setType(FsRepository.TYPE).setSettings(Settings.builder().put("location", randomAlphaOfLength(10))) + .execute( + assertNoFailureListener( + () -> masterNode.client.admin().indices().create( + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( + Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + assertNoFailureListener( + () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener(() -> createdSnapshot.set(true))))))); + + deterministicTaskQueue.runAllRunnableTasks(); + + assertTrue(createdSnapshot.get()); + SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); + final Repository repository = masterNode.repositoriesService.repository(repoName); + Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); + assertThat(snapshotIds, hasSize(1)); + + final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotIds.iterator().next()); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertThat(snapshotInfo.indices(), containsInAnyOrder(index)); + assertEquals(shards, snapshotInfo.successfulShards()); + assertEquals(0, snapshotInfo.failedShards()); + } + + private void startCluster() { final ClusterState initialClusterState = new ClusterState.Builder(ClusterName.DEFAULT).nodes(testClusterNodes.randomDiscoveryNodes()).build(); @@ -292,6 +344,8 @@ private final class TestClusterNodes { // LinkedHashMap so we have deterministic ordering when iterating over the map in tests private final Map nodes = new LinkedHashMap<>(); + private NetworkDisruption.DisruptedLinks disruptedLinks = NO_DISRUPTION; + TestClusterNodes(int masterNodes, int dataNodes) { for (int i = 0; i < masterNodes; ++i) { nodes.computeIfAbsent("node" + i, nodeName -> { @@ -390,7 +444,8 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { mockTransport = new DisruptableMockTransport(node, logger) { @Override protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { - return ConnectionStatus.CONNECTED; + return disruption.disrupt(sender.getName(), destination.getName()) + ? ConnectionStatus.DISCONNECTED : ConnectionStatus.CONNECTED; } @Override From 39337e036d01f606778c5192a17a1d62f37a6d5f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 19 Jan 2019 15:06:32 +0100 Subject: [PATCH 19/62] bck --- .../snapshots/SnapshotsServiceTests.java | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 36fedd8011b9e..855274e9fcb52 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -320,21 +320,6 @@ private Environment createEnvironment(String nodeName) { .build()); } - private TestClusterNode newMasterNode(String nodeName) throws IOException { - return newNode(nodeName, DiscoveryNode.Role.MASTER); - } - - private TestClusterNode newDataNode(String nodeName) throws IOException { - return newNode(nodeName, DiscoveryNode.Role.DATA); - } - - private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws IOException { - return new TestClusterNode( - new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(), - Collections.singleton(role), Version.CURRENT) - ); - } - private static ClusterState stateForNode(ClusterState state, DiscoveryNode node) { return ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(node.getId())).build(); } @@ -350,7 +335,7 @@ private final class TestClusterNodes { for (int i = 0; i < masterNodes; ++i) { nodes.computeIfAbsent("node" + i, nodeName -> { try { - return SnapshotsServiceTests.this.newMasterNode(nodeName); + return newMasterNode(nodeName); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -359,7 +344,7 @@ private final class TestClusterNodes { for (int i = 0; i < dataNodes; ++i) { nodes.computeIfAbsent("data-node" + i, nodeName -> { try { - return SnapshotsServiceTests.this.newDataNode(nodeName); + return newDataNode(nodeName); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -367,6 +352,24 @@ private final class TestClusterNodes { } } + private TestClusterNode newMasterNode(String nodeName) throws IOException { + return newNode(nodeName, DiscoveryNode.Role.MASTER); + } + + private TestClusterNode newDataNode(String nodeName) throws IOException { + return newNode(nodeName, DiscoveryNode.Role.DATA); + } + + private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws IOException { + return new TestClusterNode( + new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(), + Collections.singleton(role), Version.CURRENT), this::getDisruption); + } + + private NetworkDisruption.DisruptedLinks getDisruption() { + return disruptedLinks; + } + /** * Builds a {@link DiscoveryNodes} instance that has one master eligible node set as its master * by random. @@ -427,7 +430,7 @@ private final class TestClusterNode { private Coordinator coordinator; - TestClusterNode(DiscoveryNode node) throws IOException { + TestClusterNode(DiscoveryNode node, Supplier disruption) throws IOException { this.node = node; final Environment environment = createEnvironment(node.getName()); masterService = new FakeThreadPoolMasterService(node.getName(), "test", deterministicTaskQueue::scheduleNow); @@ -444,7 +447,7 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { mockTransport = new DisruptableMockTransport(node, logger) { @Override protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { - return disruption.disrupt(sender.getName(), destination.getName()) + return disruption.get().disrupt(node.getName(), destination.getName()) ? ConnectionStatus.DISCONNECTED : ConnectionStatus.CONNECTED; } From 3b19373e0c417d5bf94d31cfa4006f129ed80110 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 19 Jan 2019 15:51:30 +0100 Subject: [PATCH 20/62] works but gets stuck on recovery --- .../snapshots/SnapshotsServiceTests.java | 138 +++++++++++------- 1 file changed, 89 insertions(+), 49 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 855274e9fcb52..c9cbbab46d114 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.snapshots; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -109,11 +110,14 @@ import java.nio.file.Path; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -125,13 +129,12 @@ import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; public class SnapshotsServiceTests extends ESTestCase { - private static final Logger LOGGER = LogManager.getLogger(SnapshotsServiceTests.class); - private static final NetworkDisruption.DisruptedLinks NO_DISRUPTION = new NetworkDisruption.DisruptedLinks() { @Override public boolean disrupt(final String node1, final String node2) { @@ -206,7 +209,8 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - public void testSnapshotWithMasterFailover() { + @Repeat(iterations = 1000) + public void testSnapshotWithDataNodeDisconnects() { setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); String repoName = "repo"; @@ -229,28 +233,31 @@ public void testSnapshotWithMasterFailover() { .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener(() -> createdSnapshot.set(true))))))); - - deterministicTaskQueue.runAllRunnableTasks(); + .execute(assertNoFailureListener(() -> { + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + createdSnapshot.set(true); + })))))); + + runUntil(() -> { + if (createdSnapshot.get() == false) { + return false; + } + final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + return snapshotsInProgress.entries().isEmpty(); + }, TimeUnit.MINUTES.toMillis(5L)); assertTrue(createdSnapshot.get()); SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); - assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); + assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = masterNode.repositoriesService.repository(repoName); Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); assertThat(snapshotIds, hasSize(1)); - - final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotIds.iterator().next()); - assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); - assertThat(snapshotInfo.indices(), containsInAnyOrder(index)); - assertEquals(shards, snapshotInfo.successfulShards()); - assertEquals(0, snapshotInfo.failedShards()); } - private void startCluster() { final ClusterState initialClusterState = - new ClusterState.Builder(ClusterName.DEFAULT).nodes(testClusterNodes.randomDiscoveryNodes()).build(); + new ClusterState.Builder(ClusterName.DEFAULT).nodes(testClusterNodes.discoveryNodes()).build(); testClusterNodes.nodes.values().forEach(testClusterNode -> testClusterNode.start(initialClusterState)); deterministicTaskQueue.advanceTime(); @@ -329,7 +336,7 @@ private final class TestClusterNodes { // LinkedHashMap so we have deterministic ordering when iterating over the map in tests private final Map nodes = new LinkedHashMap<>(); - private NetworkDisruption.DisruptedLinks disruptedLinks = NO_DISRUPTION; + private DisconnectedNodes disruptedLinks = new DisconnectedNodes(); TestClusterNodes(int masterNodes, int dataNodes) { for (int i = 0; i < masterNodes; ++i) { @@ -366,16 +373,31 @@ private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws Collections.singleton(role), Version.CURRENT), this::getDisruption); } + public TestClusterNode randomDataNode() { + // Select from sorted list of data-nodes here to not have deterministic behaviour + return randomFrom( + testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) + .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) + ); + } + + public void disconnectNode(TestClusterNode node) { + disruptedLinks.disconnect(node.node.getName()); + } + + public void clearNetworkDisruptions() { + disruptedLinks.clear(); + } + private NetworkDisruption.DisruptedLinks getDisruption() { return disruptedLinks; } /** - * Builds a {@link DiscoveryNodes} instance that has one master eligible node set as its master - * by random. - * @return DiscoveryNodes with set master node + * Builds a {@link DiscoveryNodes} instance that holds the nodes in this test cluster. + * @return DiscoveryNodes */ - public DiscoveryNodes randomDiscoveryNodes() { + public DiscoveryNodes discoveryNodes() { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); nodes.values().forEach(node -> builder.add(node.node)); return builder.build(); @@ -525,39 +547,39 @@ protected void assertSnapshotOrGenericThread() { deterministicTaskQueue.getThreadPool() ); indicesClusterStateService = new IndicesClusterStateService( + settings, + indicesService, + clusterService, + threadPool, + new PeerRecoveryTargetService( + deterministicTaskQueue.getThreadPool(), transportService, recoverySettings, clusterService), + shardStateAction, + new NodeMappingRefreshAction(transportService, new MetaDataMappingService(clusterService, indicesService)), + repositoriesService, + mock(SearchService.class), + new SyncedFlushService(indicesService, clusterService, transportService, indexNameExpressionResolver), + new PeerRecoverySourceService(transportService, indicesService, recoverySettings), + snapshotShardsService, + new PrimaryReplicaSyncer( + transportService, + new TransportResyncReplicationAction( + settings, + transportService, + clusterService, + indicesService, + threadPool, + shardStateAction, + actionFilters, + indexNameExpressionResolver)), + new GlobalCheckpointSyncAction( settings, - indicesService, + transportService, clusterService, + indicesService, threadPool, - new PeerRecoveryTargetService( - deterministicTaskQueue.getThreadPool(), transportService, recoverySettings, clusterService), shardStateAction, - new NodeMappingRefreshAction(transportService, new MetaDataMappingService(clusterService, indicesService)), - repositoriesService, - mock(SearchService.class), - new SyncedFlushService(indicesService, clusterService, transportService, indexNameExpressionResolver), - new PeerRecoverySourceService(transportService, indicesService, recoverySettings), - snapshotShardsService, - new PrimaryReplicaSyncer( - transportService, - new TransportResyncReplicationAction( - settings, - transportService, - clusterService, - indicesService, - threadPool, - shardStateAction, - actionFilters, - indexNameExpressionResolver)), - new GlobalCheckpointSyncAction( - settings, - transportService, - clusterService, - indicesService, - threadPool, - shardStateAction, - actionFilters, - indexNameExpressionResolver)); + actionFilters, + indexNameExpressionResolver)); Map actions = new HashMap<>(); actions.put(CreateIndexAction.INSTANCE, new TransportCreateIndexAction( @@ -613,4 +635,22 @@ public void connectToNodes(DiscoveryNodes discoveryNodes) { coordinator.startInitialJoin(); } } + + private static final class DisconnectedNodes extends NetworkDisruption.DisruptedLinks { + + private final Set disconnected = new HashSet<>(); + + @Override + public boolean disrupt(String node1, String node2) { + return disconnected.contains(node1) || disconnected.contains(node2); + } + + public void disconnect(String node) { + disconnected.add(node); + } + + public void clear() { + disconnected.clear(); + } + } } From e5d73b3f06b5c765aae5023fa84dc03b4ec969ac Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 08:00:23 +0100 Subject: [PATCH 21/62] reproducer --- .../snapshots/SnapshotsServiceTests.java | 47 +++++++++++++------ .../MockSinglePrioritizingExecutor.java | 5 ++ .../disruption/DisruptableMockTransport.java | 1 - 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index c9cbbab46d114..8b03ac956c737 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -72,6 +71,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; @@ -101,6 +101,10 @@ import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportInterceptor; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportService; import org.junit.After; import org.junit.Before; @@ -127,7 +131,6 @@ import static java.util.Collections.emptySet; import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; -import static org.elasticsearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; @@ -135,13 +138,6 @@ public class SnapshotsServiceTests extends ESTestCase { - private static final NetworkDisruption.DisruptedLinks NO_DISRUPTION = new NetworkDisruption.DisruptedLinks() { - @Override - public boolean disrupt(final String node1, final String node2) { - return false; - } - }; - private DeterministicTaskQueue deterministicTaskQueue; private TestClusterNodes testClusterNodes; @@ -209,7 +205,7 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - @Repeat(iterations = 1000) + // ./gradlew :server:unitTest -Dtests.seed=3A3A65C12B895D9F -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotWithDataNodeDisconnects" -Dtests.security.manager=true -Dtests.locale=es-PY -Dtests.timezone=Africa/Mbabane -Dcompiler.java=11 -Druntime.java=8 public void testSnapshotWithDataNodeDisconnects() { setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); @@ -230,12 +226,14 @@ public void testSnapshotWithDataNodeDisconnects() { new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 10))), assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + deterministicTaskQueue.scheduleAt( + deterministicTaskQueue.getCurrentTimeMillis() + 20L, + () -> testClusterNodes.clearNetworkDisruptions()); createdSnapshot.set(true); })))))); @@ -245,7 +243,7 @@ public void testSnapshotWithDataNodeDisconnects() { } final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); return snapshotsInProgress.entries().isEmpty(); - }, TimeUnit.MINUTES.toMillis(5L)); + }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); @@ -487,7 +485,28 @@ protected void execute(Runnable runnable) { }; transportService = mockTransport.createTransportService( settings, deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable)), - NOOP_TRANSPORT_INTERCEPTOR, + new TransportInterceptor() { + @Override + public TransportRequestHandler interceptHandler(String action, String executor, + boolean forceExecution, TransportRequestHandler actualHandler) { + if (action.startsWith("internal:index/shard/recovery")) { + return (request, channel, task) -> deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 20L, new AbstractRunnable() { + + @Override + protected void doRun() throws Exception { + channel.sendResponse(new TransportException("Recovery not implemented yet")); + } + + @Override + public void onFailure(final Exception e) { + throw new AssertionError(e); + } + }); + } else { + return actualHandler; + } + } + }, a -> node, null, emptySet() ); final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index cc21fef5f5559..7e37871a2bcb0 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -53,6 +53,11 @@ protected void afterExecute(Runnable r, Throwable t) { throw new KillWorkerError(); } + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + private static final class KillWorkerError extends Error { } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java index 2a1101c6d7986..b728a1e981313 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java @@ -72,7 +72,6 @@ protected final void execute(String action, Runnable runnable) { if (action.equals(HANDSHAKE_ACTION_NAME)) { runnable.run(); } else { - execute(runnable); } } From ecdd36cc4812b64733ca201ec828f8804f407ace Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 09:04:47 +0100 Subject: [PATCH 22/62] fixed checkstyle --- .../snapshots/SnapshotsServiceTests.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 8b03ac956c737..767c83770e73c 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -205,7 +205,9 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - // ./gradlew :server:unitTest -Dtests.seed=3A3A65C12B895D9F -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotWithDataNodeDisconnects" -Dtests.security.manager=true -Dtests.locale=es-PY -Dtests.timezone=Africa/Mbabane -Dcompiler.java=11 -Druntime.java=8 + // ./gradlew :server:unitTest -Dtests.seed=3A3A65C12B895D9F -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests \ + // -Dtests.method="testSnapshotWithDataNodeDisconnects" -Dtests.security.manager=true -Dtests.locale=es-PY \ + // -Dtests.timezone=Africa/Mbabane -Dcompiler.java=11 -Druntime.java=8 public void testSnapshotWithDataNodeDisconnects() { setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); @@ -230,7 +232,8 @@ public void testSnapshotWithDataNodeDisconnects() { assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); deterministicTaskQueue.scheduleAt( deterministicTaskQueue.getCurrentTimeMillis() + 20L, () -> testClusterNodes.clearNetworkDisruptions()); @@ -490,18 +493,19 @@ protected void execute(Runnable runnable) { public TransportRequestHandler interceptHandler(String action, String executor, boolean forceExecution, TransportRequestHandler actualHandler) { if (action.startsWith("internal:index/shard/recovery")) { - return (request, channel, task) -> deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 20L, new AbstractRunnable() { - - @Override - protected void doRun() throws Exception { - channel.sendResponse(new TransportException("Recovery not implemented yet")); - } - - @Override - public void onFailure(final Exception e) { - throw new AssertionError(e); - } - }); + return (request, channel, task) -> deterministicTaskQueue.scheduleAt( + deterministicTaskQueue.getCurrentTimeMillis() + 20L, + new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + channel.sendResponse(new TransportException("Recovery not implemented yet")); + } + + @Override + public void onFailure(final Exception e) { + throw new AssertionError(e); + } + }); } else { return actualHandler; } From 9e389bdbcff7a90c2360f2e04c207f851073dccc Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 09:26:09 +0100 Subject: [PATCH 23/62] Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its" This reverts commit d1f36d614f956f438bb3887602a644e8bf065a1a, reversing changes made to 263c525d6e116942c0237168ac886f888b6611f8. --- .../routing/RoutingChangesObserver.java | 49 ------ .../routing/allocation/AllocationService.java | 19 +-- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ------------------ 5 files changed, 5 insertions(+), 219 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 0c0d3c5a099f9..883b4c22f7fc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.index.shard.ShardId; - /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -213,51 +211,4 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } - - abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { - - @Override - public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { - onChanged(unassignedShard.shardId()); - } - - @Override - public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { - onChanged(initializingShard.shardId()); - } - @Override - public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { - onChanged(startedShard.shardId()); - } - @Override - public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { - onChanged(unassignedShard.shardId()); - } - @Override - public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { - onChanged(failedShard.shardId()); - } - @Override - public void relocationCompleted(ShardRouting removedRelocationSource) { - onChanged(removedRelocationSource.shardId()); - } - @Override - public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { - onChanged(removedReplicaRelocationSource.shardId()); - } - @Override - public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { - onChanged(startedPrimaryShard.shardId()); - } - @Override - public void replicaPromoted(ShardRouting replicaShard) { - onChanged(replicaShard.shardId()); - } - @Override - public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { - onChanged(oldReplica.shardId()); - } - - protected abstract void onChanged(ShardId shardId); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 389eec6ed5a87..59f43a193ddc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -137,29 +136,15 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); - ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); + newStateBuilder.customs(customsBuilder.build()); } } - final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - SnapshotsInProgress updatedSnapshotsInProgress = - allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); - if (updatedSnapshotsInProgress != snapshotsInProgress) { - if (customsBuilder == null) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); - } - customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); - } - } - if (customsBuilder != null) { - newStateBuilder.customs(customsBuilder.build()); - } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index f178ca18845ae..e0be712a230c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -39,7 +38,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -78,11 +76,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); - private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater ); + /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -253,14 +251,6 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } - /** - * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes - */ - public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, - RoutingTable newRoutingTable) { - return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); - } - /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 9a3c2c4284397..6b7b506114361 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -313,7 +313,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to start + // We have new shards to starts if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 241d19a3537b2..86ed2095433b2 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -46,7 +45,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -678,28 +676,12 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); - // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here - assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } - private boolean assertConsistency(ClusterState state) { - SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - assert snapshotsInProgress == updateWithRoutingTable( - snapshotsInProgress.entries().stream().flatMap(entry -> { - Iterable iterable = () -> entry.shards().keysIt(); - return StreamSupport.stream(iterable.spliterator(), false); - }).collect(Collectors.toSet()), - snapshotsInProgress, state.routingTable() - ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; - } - return true; - } - /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1594,126 +1576,4 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } - - public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { - - private final Set shardChanges = new HashSet<>(); - - public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { - return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); - } - - @Override - protected void onChanged(ShardId shardId) { - shardChanges.add(shardId); - } - } - - private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, - RoutingTable newRoutingTable) { - if (oldSnapshot == null || shardIds.isEmpty()) { - return oldSnapshot; - } - List entries = new ArrayList<>(); - boolean snapshotsInProgressChanged = false; - for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { - ImmutableOpenMap.Builder shardsBuilder = null; - for (ShardId shardId : shardIds) { - final ImmutableOpenMap shards = entry.shards(); - final ShardSnapshotStatus currentStatus = shards.get(shardId); - if (currentStatus != null && currentStatus.state().completed() == false) { - IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); - assert routingTable != null; - final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) - .map(IndexShardRoutingTable::primaryShard) - .map( - primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) - ) - .orElse(failedStatus(null, "missing shard")); - if (newStatus != currentStatus) { - if (shardsBuilder == null) { - shardsBuilder = ImmutableOpenMap.builder(shards); - } - shardsBuilder.put(shardId, newStatus); - } - } - } - if (shardsBuilder == null) { - entries.add(entry); - } else { - snapshotsInProgressChanged = true; - ImmutableOpenMap shards = shardsBuilder.build(); - entries.add( - new SnapshotsInProgress.Entry( - entry, - completed(shards.values()) ? State.SUCCESS : entry.state(), - shards - ) - ); - } - } - if (snapshotsInProgressChanged) { - return new SnapshotsInProgress(entries); - } - return oldSnapshot; - } - - private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, - final ShardRouting primaryShardRouting) { - final State currentState = currentStatus.state(); - final ShardSnapshotStatus newStatus; - if (primaryShardRouting.active() == false) { - if (primaryShardRouting.initializing() && currentState == State.WAITING) { - newStatus = currentStatus; - } else { - newStatus = failedStatus( - primaryShardRouting.currentNodeId(), - primaryShardRouting.unassignedInfo().getReason().toString() - ); - } - } else if (primaryShardRouting.started()) { - switch (currentState) { - case WAITING: - newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); - break; - case INIT: { - String currentNodeId = currentStatus.nodeId(); - assert currentNodeId != null; - if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - } - case ABORTED: - String currentNodeId = currentStatus.nodeId(); - if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - default: - newStatus = currentStatus; - break; - } - } else { - assert primaryShardRouting.relocating(); - if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { - newStatus = failedStatus(currentStatus.nodeId()); - } else { - newStatus = currentStatus; - } - } - return newStatus; - } - - private static ShardSnapshotStatus failedStatus(String nodeId) { - return failedStatus(nodeId, "shard failed"); - } - - private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { - return new ShardSnapshotStatus(nodeId, State.FAILED, reason); - } } From 4b618a62439611e5d96f6cc89573ecead362d043 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 13:37:15 +0100 Subject: [PATCH 24/62] bck --- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 767c83770e73c..f6991d66a7e04 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -470,6 +470,9 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { mockTransport = new DisruptableMockTransport(node, logger) { @Override protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { + if (node.getName().equals(destination.getName())) { + return ConnectionStatus.CONNECTED; + } return disruption.get().disrupt(node.getName(), destination.getName()) ? ConnectionStatus.DISCONNECTED : ConnectionStatus.CONNECTED; } @@ -498,7 +501,7 @@ public TransportRequestHandler interceptHandler( new AbstractRunnable() { @Override protected void doRun() throws Exception { - channel.sendResponse(new TransportException("Recovery not implemented yet")); + channel.sendResponse(new TransportException(new IOException("failed to recover shard"))); } @Override From 683c985d592ab5cea49947460f87420aa22a0338 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 15:10:04 +0100 Subject: [PATCH 25/62] nicer --- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 5 +++++ .../cluster/coordination/MockSinglePrioritizingExecutor.java | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index f6991d66a7e04..801c07ca574f3 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -384,10 +384,15 @@ public TestClusterNode randomDataNode() { public void disconnectNode(TestClusterNode node) { disruptedLinks.disconnect(node.node.getName()); + testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().disconnectFromNode(node.node)); } public void clearNetworkDisruptions() { disruptedLinks.clear(); + disruptedLinks.disconnected.forEach(nodeName -> { + final DiscoveryNode node = testClusterNodes.nodes.get(nodeName).node; + testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().openConnection(node, null)); + }); } private NetworkDisruption.DisruptedLinks getDisruption() { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 7e37871a2bcb0..cc21fef5f5559 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -53,11 +53,6 @@ protected void afterExecute(Runnable r, Throwable t) { throw new KillWorkerError(); } - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) { - return true; - } - private static final class KillWorkerError extends Error { } } From a520aa141fcc84758e81572e5bf0075e5439faeb Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 16:11:35 +0100 Subject: [PATCH 26/62] proper disconnects --- .../snapshots/SnapshotsServiceTests.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 801c07ca574f3..c57ea9f1aebec 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -205,10 +205,8 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - // ./gradlew :server:unitTest -Dtests.seed=3A3A65C12B895D9F -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests \ - // -Dtests.method="testSnapshotWithDataNodeDisconnects" -Dtests.security.manager=true -Dtests.locale=es-PY \ - // -Dtests.timezone=Africa/Mbabane -Dcompiler.java=11 -Druntime.java=8 - public void testSnapshotWithDataNodeDisconnects() { + // -Dtests.seed=1BA5E8C381FEDE95 + public void testSnapshotWithNodeDisconnects() { setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); String repoName = "repo"; @@ -228,12 +226,12 @@ public void testSnapshotWithDataNodeDisconnects() { new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 10))), + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)), assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + () -> testClusterNodes.disconnectNode(testClusterNodes.randomNode())); deterministicTaskQueue.scheduleAt( deterministicTaskQueue.getCurrentTimeMillis() + 20L, () -> testClusterNodes.clearNetworkDisruptions()); @@ -374,11 +372,10 @@ private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws Collections.singleton(role), Version.CURRENT), this::getDisruption); } - public TestClusterNode randomDataNode() { + public TestClusterNode randomNode() { // Select from sorted list of data-nodes here to not have deterministic behaviour return randomFrom( - testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) - .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) + testClusterNodes.nodes.values().stream().sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) ); } From 6237813f721afb1025343566b8693d5a893b102f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 16:11:44 +0100 Subject: [PATCH 27/62] Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its"" This reverts commit 9e389bdbcff7a90c2360f2e04c207f851073dccc. --- .../routing/RoutingChangesObserver.java | 49 ++++++ .../routing/allocation/AllocationService.java | 19 ++- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ++++++++++++++++++ 5 files changed, 219 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 883b4c22f7fc0..0c0d3c5a099f9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,6 +19,8 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.index.shard.ShardId; + /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -211,4 +213,51 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } + + abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { + + @Override + public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { + onChanged(unassignedShard.shardId()); + } + + @Override + public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { + onChanged(initializingShard.shardId()); + } + @Override + public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { + onChanged(startedShard.shardId()); + } + @Override + public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { + onChanged(unassignedShard.shardId()); + } + @Override + public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { + onChanged(failedShard.shardId()); + } + @Override + public void relocationCompleted(ShardRouting removedRelocationSource) { + onChanged(removedRelocationSource.shardId()); + } + @Override + public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { + onChanged(removedReplicaRelocationSource.shardId()); + } + @Override + public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { + onChanged(startedPrimaryShard.shardId()); + } + @Override + public void replicaPromoted(ShardRouting replicaShard) { + onChanged(replicaShard.shardId()); + } + @Override + public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { + onChanged(oldReplica.shardId()); + } + + protected abstract void onChanged(ShardId shardId); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 59f43a193ddc8..389eec6ed5a87 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -136,15 +137,29 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); + ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); - newStateBuilder.customs(customsBuilder.build()); } } + final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + SnapshotsInProgress updatedSnapshotsInProgress = + allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); + if (updatedSnapshotsInProgress != snapshotsInProgress) { + if (customsBuilder == null) { + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + } + customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); + } + } + if (customsBuilder != null) { + newStateBuilder.customs(customsBuilder.build()); + } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index e0be712a230c7..f178ca18845ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -76,11 +78,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); + private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater ); - /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -251,6 +253,14 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } + /** + * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes + */ + public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, + RoutingTable newRoutingTable) { + return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); + } + /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 6b7b506114361..9a3c2c4284397 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -313,7 +313,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to starts + // We have new shards to start if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 86ed2095433b2..241d19a3537b2 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -45,6 +46,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -676,12 +678,28 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); + // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here + assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } + private boolean assertConsistency(ClusterState state) { + SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + assert snapshotsInProgress == updateWithRoutingTable( + snapshotsInProgress.entries().stream().flatMap(entry -> { + Iterable iterable = () -> entry.shards().keysIt(); + return StreamSupport.stream(iterable.spliterator(), false); + }).collect(Collectors.toSet()), + snapshotsInProgress, state.routingTable() + ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; + } + return true; + } + /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1576,4 +1594,126 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } + + public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { + + private final Set shardChanges = new HashSet<>(); + + public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { + return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); + } + + @Override + protected void onChanged(ShardId shardId) { + shardChanges.add(shardId); + } + } + + private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, + RoutingTable newRoutingTable) { + if (oldSnapshot == null || shardIds.isEmpty()) { + return oldSnapshot; + } + List entries = new ArrayList<>(); + boolean snapshotsInProgressChanged = false; + for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { + ImmutableOpenMap.Builder shardsBuilder = null; + for (ShardId shardId : shardIds) { + final ImmutableOpenMap shards = entry.shards(); + final ShardSnapshotStatus currentStatus = shards.get(shardId); + if (currentStatus != null && currentStatus.state().completed() == false) { + IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); + assert routingTable != null; + final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) + .map(IndexShardRoutingTable::primaryShard) + .map( + primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) + ) + .orElse(failedStatus(null, "missing shard")); + if (newStatus != currentStatus) { + if (shardsBuilder == null) { + shardsBuilder = ImmutableOpenMap.builder(shards); + } + shardsBuilder.put(shardId, newStatus); + } + } + } + if (shardsBuilder == null) { + entries.add(entry); + } else { + snapshotsInProgressChanged = true; + ImmutableOpenMap shards = shardsBuilder.build(); + entries.add( + new SnapshotsInProgress.Entry( + entry, + completed(shards.values()) ? State.SUCCESS : entry.state(), + shards + ) + ); + } + } + if (snapshotsInProgressChanged) { + return new SnapshotsInProgress(entries); + } + return oldSnapshot; + } + + private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, + final ShardRouting primaryShardRouting) { + final State currentState = currentStatus.state(); + final ShardSnapshotStatus newStatus; + if (primaryShardRouting.active() == false) { + if (primaryShardRouting.initializing() && currentState == State.WAITING) { + newStatus = currentStatus; + } else { + newStatus = failedStatus( + primaryShardRouting.currentNodeId(), + primaryShardRouting.unassignedInfo().getReason().toString() + ); + } + } else if (primaryShardRouting.started()) { + switch (currentState) { + case WAITING: + newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); + break; + case INIT: { + String currentNodeId = currentStatus.nodeId(); + assert currentNodeId != null; + if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + } + case ABORTED: + String currentNodeId = currentStatus.nodeId(); + if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + default: + newStatus = currentStatus; + break; + } + } else { + assert primaryShardRouting.relocating(); + if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { + newStatus = failedStatus(currentStatus.nodeId()); + } else { + newStatus = currentStatus; + } + } + return newStatus; + } + + private static ShardSnapshotStatus failedStatus(String nodeId) { + return failedStatus(nodeId, "shard failed"); + } + + private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { + return new ShardSnapshotStatus(nodeId, State.FAILED, reason); + } } From 1c0a3b98a97f0a26da5362eb2e9a1f83cfe00c18 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 16:12:19 +0100 Subject: [PATCH 28/62] Revert "Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its""" This reverts commit 6237813f721afb1025343566b8693d5a893b102f. --- .../routing/RoutingChangesObserver.java | 49 ------ .../routing/allocation/AllocationService.java | 19 +-- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ------------------ 5 files changed, 5 insertions(+), 219 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 0c0d3c5a099f9..883b4c22f7fc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.index.shard.ShardId; - /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -213,51 +211,4 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } - - abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { - - @Override - public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { - onChanged(unassignedShard.shardId()); - } - - @Override - public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { - onChanged(initializingShard.shardId()); - } - @Override - public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { - onChanged(startedShard.shardId()); - } - @Override - public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { - onChanged(unassignedShard.shardId()); - } - @Override - public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { - onChanged(failedShard.shardId()); - } - @Override - public void relocationCompleted(ShardRouting removedRelocationSource) { - onChanged(removedRelocationSource.shardId()); - } - @Override - public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { - onChanged(removedReplicaRelocationSource.shardId()); - } - @Override - public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { - onChanged(startedPrimaryShard.shardId()); - } - @Override - public void replicaPromoted(ShardRouting replicaShard) { - onChanged(replicaShard.shardId()); - } - @Override - public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { - onChanged(oldReplica.shardId()); - } - - protected abstract void onChanged(ShardId shardId); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 389eec6ed5a87..59f43a193ddc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -137,29 +136,15 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); - ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); + newStateBuilder.customs(customsBuilder.build()); } } - final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - SnapshotsInProgress updatedSnapshotsInProgress = - allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); - if (updatedSnapshotsInProgress != snapshotsInProgress) { - if (customsBuilder == null) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); - } - customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); - } - } - if (customsBuilder != null) { - newStateBuilder.customs(customsBuilder.build()); - } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index f178ca18845ae..e0be712a230c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -39,7 +38,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -78,11 +76,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); - private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater ); + /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -253,14 +251,6 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } - /** - * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes - */ - public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, - RoutingTable newRoutingTable) { - return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); - } - /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 9a3c2c4284397..6b7b506114361 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -313,7 +313,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to start + // We have new shards to starts if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 241d19a3537b2..86ed2095433b2 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -46,7 +45,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -678,28 +676,12 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); - // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here - assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } - private boolean assertConsistency(ClusterState state) { - SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - assert snapshotsInProgress == updateWithRoutingTable( - snapshotsInProgress.entries().stream().flatMap(entry -> { - Iterable iterable = () -> entry.shards().keysIt(); - return StreamSupport.stream(iterable.spliterator(), false); - }).collect(Collectors.toSet()), - snapshotsInProgress, state.routingTable() - ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; - } - return true; - } - /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1594,126 +1576,4 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } - - public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { - - private final Set shardChanges = new HashSet<>(); - - public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { - return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); - } - - @Override - protected void onChanged(ShardId shardId) { - shardChanges.add(shardId); - } - } - - private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, - RoutingTable newRoutingTable) { - if (oldSnapshot == null || shardIds.isEmpty()) { - return oldSnapshot; - } - List entries = new ArrayList<>(); - boolean snapshotsInProgressChanged = false; - for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { - ImmutableOpenMap.Builder shardsBuilder = null; - for (ShardId shardId : shardIds) { - final ImmutableOpenMap shards = entry.shards(); - final ShardSnapshotStatus currentStatus = shards.get(shardId); - if (currentStatus != null && currentStatus.state().completed() == false) { - IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); - assert routingTable != null; - final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) - .map(IndexShardRoutingTable::primaryShard) - .map( - primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) - ) - .orElse(failedStatus(null, "missing shard")); - if (newStatus != currentStatus) { - if (shardsBuilder == null) { - shardsBuilder = ImmutableOpenMap.builder(shards); - } - shardsBuilder.put(shardId, newStatus); - } - } - } - if (shardsBuilder == null) { - entries.add(entry); - } else { - snapshotsInProgressChanged = true; - ImmutableOpenMap shards = shardsBuilder.build(); - entries.add( - new SnapshotsInProgress.Entry( - entry, - completed(shards.values()) ? State.SUCCESS : entry.state(), - shards - ) - ); - } - } - if (snapshotsInProgressChanged) { - return new SnapshotsInProgress(entries); - } - return oldSnapshot; - } - - private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, - final ShardRouting primaryShardRouting) { - final State currentState = currentStatus.state(); - final ShardSnapshotStatus newStatus; - if (primaryShardRouting.active() == false) { - if (primaryShardRouting.initializing() && currentState == State.WAITING) { - newStatus = currentStatus; - } else { - newStatus = failedStatus( - primaryShardRouting.currentNodeId(), - primaryShardRouting.unassignedInfo().getReason().toString() - ); - } - } else if (primaryShardRouting.started()) { - switch (currentState) { - case WAITING: - newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); - break; - case INIT: { - String currentNodeId = currentStatus.nodeId(); - assert currentNodeId != null; - if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - } - case ABORTED: - String currentNodeId = currentStatus.nodeId(); - if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - default: - newStatus = currentStatus; - break; - } - } else { - assert primaryShardRouting.relocating(); - if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { - newStatus = failedStatus(currentStatus.nodeId()); - } else { - newStatus = currentStatus; - } - } - return newStatus; - } - - private static ShardSnapshotStatus failedStatus(String nodeId) { - return failedStatus(nodeId, "shard failed"); - } - - private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { - return new ShardSnapshotStatus(nodeId, State.FAILED, reason); - } } From d2232ce1c6c273662f123df57b215d72895cbf13 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 17:28:24 +0100 Subject: [PATCH 29/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index c57ea9f1aebec..00d8da0371394 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -205,7 +205,6 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - // -Dtests.seed=1BA5E8C381FEDE95 public void testSnapshotWithNodeDisconnects() { setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); @@ -230,11 +229,20 @@ public void testSnapshotWithNodeDisconnects() { assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { - deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomNode())); - deterministicTaskQueue.scheduleAt( - deterministicTaskQueue.getCurrentTimeMillis() + 20L, - () -> testClusterNodes.clearNetworkDisruptions()); + if (randomBoolean()) { + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + } + final boolean stoppedMasterNode = randomBoolean(); + if (stoppedMasterNode) { + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode())); + } + if (stoppedMasterNode || randomBoolean()) { + deterministicTaskQueue.scheduleAt( + deterministicTaskQueue.getCurrentTimeMillis() + 20L, + () -> testClusterNodes.clearNetworkDisruptions()); + } createdSnapshot.set(true); })))))); @@ -285,10 +293,10 @@ private void startCluster() { private void runUntil(Supplier fulfilled, long timeout) { final long start = deterministicTaskQueue.getCurrentTimeMillis(); while (timeout > deterministicTaskQueue.getCurrentTimeMillis() - start) { - deterministicTaskQueue.runAllRunnableTasks(); if (fulfilled.get()) { return; } + deterministicTaskQueue.runAllRunnableTasks(); deterministicTaskQueue.advanceTime(); } fail("Condition wasn't fulfilled."); @@ -372,10 +380,19 @@ private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws Collections.singleton(role), Version.CURRENT), this::getDisruption); } - public TestClusterNode randomNode() { + public TestClusterNode randomMasterNode() { + // Select from sorted list of data-nodes here to not have deterministic behaviour + return randomFrom( + testClusterNodes.nodes.values().stream().filter(n -> n.node.isMasterNode()) + .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) + ); + } + + public TestClusterNode randomDataNode() { // Select from sorted list of data-nodes here to not have deterministic behaviour return randomFrom( - testClusterNodes.nodes.values().stream().sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) + testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) + .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) ); } @@ -651,10 +668,17 @@ public void start(ClusterState initialState) { @Override public void connectToNodes(DiscoveryNodes discoveryNodes) { // override this method as it does blocking calls + boolean callSuper = true; for (final DiscoveryNode node : discoveryNodes) { - transportService.connectToNode(node); + try { + transportService.connectToNode(node); + } catch (Exception e) { + callSuper = false; + } + } + if (callSuper) { + super.connectToNodes(discoveryNodes); } - super.connectToNodes(discoveryNodes); } }); clusterService.getClusterApplierService().start(); From e493b415285cdbe16788eb6f8d876d5c63c74ab7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 21:44:46 +0100 Subject: [PATCH 30/62] still passes --- .../snapshots/SnapshotsServiceTests.java | 60 ++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 00d8da0371394..dafa15b790b19 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -206,7 +206,9 @@ public void testSuccessfulSnapshot() { } public void testSnapshotWithNodeDisconnects() { - setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); + final int masterNodes = randomFrom(1, 3, 5); + final int dataNodes = randomIntBetween(2, 10); + setupTestCluster(masterNodes, dataNodes); String repoName = "repo"; String snapshotName = "snapshot"; @@ -225,33 +227,46 @@ public void testSnapshotWithNodeDisconnects() { new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)), + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), assertNoFailureListener( - () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener(() -> { - if (randomBoolean()) { - deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); - } - final boolean stoppedMasterNode = randomBoolean(); - if (stoppedMasterNode) { - deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode())); - } - if (stoppedMasterNode || randomBoolean()) { - deterministicTaskQueue.scheduleAt( - deterministicTaskQueue.getCurrentTimeMillis() + 20L, - () -> testClusterNodes.clearNetworkDisruptions()); - } - createdSnapshot.set(true); - })))))); + () -> { + for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + } + if (randomBoolean()) { + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + } + masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener(() -> { + for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + } + final boolean disconnectedMaster = randomBoolean(); + if (disconnectedMaster) { + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode())); + } + if (disconnectedMaster || randomBoolean()) { + deterministicTaskQueue.scheduleAt( + deterministicTaskQueue.getCurrentTimeMillis() + + randomLongBetween(0L, TimeUnit.MINUTES.toMillis(10L)), + () -> testClusterNodes.clearNetworkDisruptions()); + } + if (randomBoolean()) { + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + } + createdSnapshot.set(true); + })); + })))); runUntil(() -> { if (createdSnapshot.get() == false) { return false; } final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); - return snapshotsInProgress.entries().isEmpty(); + return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); @@ -397,6 +412,9 @@ public TestClusterNode randomDataNode() { } public void disconnectNode(TestClusterNode node) { + if (disruptedLinks.disconnected.contains(node.node.getName())) { + return; + } disruptedLinks.disconnect(node.node.getName()); testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().disconnectFromNode(node.node)); } From 3a9a25aff1c0870bb4652b17ea4484105c44c971 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 20 Jan 2019 22:53:07 +0100 Subject: [PATCH 31/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 74 ++++++++++++++----- .../MockSinglePrioritizingExecutor.java | 5 ++ 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index dafa15b790b19..5c9748212865e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.snapshots; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -98,6 +99,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.TestCluster; import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.threadpool.ThreadPool; @@ -153,15 +155,7 @@ public void createServices() { @After public void stopServices() { - testClusterNodes.nodes.values().forEach( - n -> { - n.indicesService.close(); - n.clusterService.close(); - n.indicesClusterStateService.close(); - n.nodeEnv.close(); - n.coordinator.close(); - } - ); + testClusterNodes.nodes.values().forEach(TestClusterNode::stop); } public void testSuccessfulSnapshot() { @@ -205,6 +199,7 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } + @Repeat(iterations = 1000) public void testSnapshotWithNodeDisconnects() { final int masterNodes = randomFrom(1, 3, 5); final int dataNodes = randomIntBetween(2, 10); @@ -232,7 +227,7 @@ public void testSnapshotWithNodeDisconnects() { () -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + () -> testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n))); } if (randomBoolean()) { deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); @@ -241,12 +236,24 @@ public void testSnapshotWithNodeDisconnects() { .execute(assertNoFailureListener(() -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomDataNode())); + () -> { + if (randomBoolean()) { + testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n)); + } else { + testClusterNodes.randomDataNode().ifPresent(TestClusterNode::restart); + } + }); } final boolean disconnectedMaster = randomBoolean(); if (disconnectedMaster) { deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode())); + () -> { + if (randomBoolean()) { + testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode()); + } else { + testClusterNodes.randomDataNode().ifPresent(TestClusterNode::restart); + } + }); } if (disconnectedMaster || randomBoolean()) { deterministicTaskQueue.scheduleAt( @@ -403,12 +410,11 @@ public TestClusterNode randomMasterNode() { ); } - public TestClusterNode randomDataNode() { + public Optional randomDataNode() { // Select from sorted list of data-nodes here to not have deterministic behaviour - return randomFrom( - testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) - .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) - ); + final List dataNodes = testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) + .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()); + return dataNodes.isEmpty() ? Optional.empty() : Optional.ofNullable(randomFrom(dataNodes)); } public void disconnectNode(TestClusterNode node) { @@ -488,9 +494,12 @@ private final class TestClusterNode { private final ThreadPool threadPool; + private final Supplier disruption; + private Coordinator coordinator; TestClusterNode(DiscoveryNode node, Supplier disruption) throws IOException { + this.disruption = disruption; this.node = node; final Environment environment = createEnvironment(node.getName()); masterService = new FakeThreadPoolMasterService(node.getName(), "test", deterministicTaskQueue::scheduleNow); @@ -665,13 +674,42 @@ allocationService, new AliasValidator(), environment, indexScopedSettings, client.initialize(actions, () -> clusterService.localNode().getId(), transportService.getRemoteClusterService()); } + public void restart() { + testClusterNodes.disconnectNode(this); + final ClusterState oldState = this.clusterService.state(); + stop(); + testClusterNodes.nodes.remove(node.getName()); + deterministicTaskQueue.scheduleAt(randomFrom(0L, deterministicTaskQueue.getCurrentTimeMillis()), () -> { + try { + final TestClusterNode restartedNode = + new TestClusterNode( + new DiscoveryNode(node.getName(), node.getId(), buildNewFakeTransportAddress(), emptyMap(), + node.getRoles(), Version.CURRENT), disruption); + testClusterNodes.nodes.put(node.getName(), restartedNode); + restartedNode.start(oldState); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + public void stop() { + indicesService.close(); + clusterService.close(); + indicesClusterStateService.close(); + if (coordinator != null) { + coordinator.close(); + } + nodeEnv.close(); + } + public void start(ClusterState initialState) { transportService.start(); transportService.acceptIncomingRequests(); snapshotsService.start(); snapshotShardsService.start(); final CoordinationState.PersistedState persistedState = - new InMemoryPersistedState(0L, stateForNode(initialState, node)); + new InMemoryPersistedState(initialState.term(), stateForNode(initialState, node)); coordinator = new Coordinator(node.getName(), clusterService.getSettings(), clusterService.getClusterSettings(), transportService, namedWriteableRegistry, allocationService, masterService, () -> persistedState, diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index cc21fef5f5559..7e37871a2bcb0 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -53,6 +53,11 @@ protected void afterExecute(Runnable r, Throwable t) { throw new KillWorkerError(); } + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + private static final class KillWorkerError extends Error { } } From 801376f99a831ec25f0520e43d6f542a45678a58 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 11:46:04 +0100 Subject: [PATCH 32/62] relocation it --- .../snapshots/SnapshotsServiceTests.java | 196 ++++++++++++++++-- .../disruption/DisruptableMockTransport.java | 7 +- .../test/transport/MockTransport.java | 14 +- 3 files changed, 192 insertions(+), 25 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 5c9748212865e..f2691aff9fadf 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -28,15 +28,25 @@ import org.elasticsearch.action.admin.cluster.bootstrap.BootstrapConfiguration; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryAction; import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; +import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; +import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; @@ -61,11 +71,14 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingService; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -79,6 +92,7 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.gateway.MetaStateService; +import org.elasticsearch.gateway.TransportNodesListGatewayStartedShards; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction; import org.elasticsearch.index.shard.PrimaryReplicaSyncer; @@ -99,7 +113,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.TestCluster; import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.threadpool.ThreadPool; @@ -126,8 +139,10 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -199,11 +214,9 @@ public void testSuccessfulSnapshot() { assertEquals(0, snapshotInfo.failedShards()); } - @Repeat(iterations = 1000) public void testSnapshotWithNodeDisconnects() { - final int masterNodes = randomFrom(1, 3, 5); final int dataNodes = randomIntBetween(2, 10); - setupTestCluster(masterNodes, dataNodes); + setupTestCluster(randomFrom(1, 3, 5), dataNodes); String repoName = "repo"; String snapshotName = "snapshot"; @@ -214,11 +227,12 @@ public void testSnapshotWithNodeDisconnects() { TestClusterNode masterNode = testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state()); final AtomicBoolean createdSnapshot = new AtomicBoolean(); - masterNode.client.admin().cluster().preparePutRepository(repoName) + final AdminClient masterAdminClient = masterNode.client.admin(); + masterAdminClient.cluster().preparePutRepository(repoName) .setType(FsRepository.TYPE).setSettings(Settings.builder().put("location", randomAlphaOfLength(10))) .execute( assertNoFailureListener( - () -> masterNode.client.admin().indices().create( + () -> masterAdminClient.indices().create( new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( Settings.builder() .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) @@ -226,23 +240,15 @@ public void testSnapshotWithNodeDisconnects() { assertNoFailureListener( () -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { - deterministicTaskQueue.scheduleNow( - () -> testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n))); + deterministicTaskQueue.scheduleNow(this::disconnectRandomDataNode); } if (randomBoolean()) { - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + scheduleClearDisruptionNow(); } - masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { - deterministicTaskQueue.scheduleNow( - () -> { - if (randomBoolean()) { - testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n)); - } else { - testClusterNodes.randomDataNode().ifPresent(TestClusterNode::restart); - } - }); + deterministicTaskQueue.scheduleNow(this::disconnectOrRestartDataNode); } final boolean disconnectedMaster = randomBoolean(); if (disconnectedMaster) { @@ -260,9 +266,8 @@ public void testSnapshotWithNodeDisconnects() { deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0L, TimeUnit.MINUTES.toMillis(10L)), () -> testClusterNodes.clearNetworkDisruptions()); - } - if (randomBoolean()) { - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + } else if (randomBoolean()) { + scheduleClearDisruptionNow(); } createdSnapshot.set(true); })); @@ -284,6 +289,109 @@ public void testSnapshotWithNodeDisconnects() { assertThat(snapshotIds, hasSize(1)); } + @Repeat(iterations = 1000) + public void testSnapshotPrimaryRelocations() { + setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); + + String repoName = "repo"; + String snapshotName = "snapshot"; + final String index = "test"; + + final int shards = randomIntBetween(1, 10); + + final TestClusterNode masterNode = + testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state()); + final AtomicBoolean createdSnapshot = new AtomicBoolean(); + final AdminClient masterAdminClient = masterNode.client.admin(); + masterAdminClient.cluster().preparePutRepository(repoName) + .setType(FsRepository.TYPE).setSettings(Settings.builder().put("location", randomAlphaOfLength(10))) + .execute( + assertNoFailureListener( + () -> masterAdminClient.indices().create( + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( + Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + assertNoFailureListener( + () -> masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( + clusterStateResponse -> { + RoutingTable table = clusterStateResponse.getState().routingTable(); + final TestClusterNode currentPrimaryNode = + testClusterNodes.nodeById(table.allShards(index).get(0).currentNodeId()); + final TestClusterNode otherNode = testClusterNodes.randomDataNode(currentPrimaryNode.node.getName()) + .orElseThrow(() -> new AssertionError("Could not find another data node.")); + deterministicTaskQueue.scheduleNow(() -> { + currentPrimaryNode.stop(); + Runnable assignStalePrimary = () -> masterAdminClient.cluster().reroute( + new ClusterRerouteRequest().add( + new AllocateEmptyPrimaryAllocationCommand( + index, 0, otherNode.node.getName(), true + ) + ), + new ActionListener() { + @Override + public void onResponse(final ClusterRerouteResponse clusterRerouteResponse) { + masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener( + () -> createdSnapshot.set(true))); + } + + @Override + public void onFailure(final Exception e) { + masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener( + () -> createdSnapshot.set(true))); + } + }); + deterministicTaskQueue.scheduleNow( + new Runnable() { + @Override + public void run() { + if (masterNode.clusterService.state().routingTable().allShards(index).get(0).unassigned() || rarely()) { + assignStalePrimary.run(); + } else { + if (frequently()) { + deterministicTaskQueue.scheduleNow(this); + } else { + deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 10L, this); + } + } + } + } + ); + }); + } + )))))); + + runUntil(() -> { + final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); + }, TimeUnit.MINUTES.toMillis(20L)); + + assertTrue(createdSnapshot.get()); + SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + assertThat(finalSnapshotsInProgress.entries(), empty()); + final Repository repository = masterNode.repositoriesService.repository(repoName); + Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); + assertThat(snapshotIds, hasSize(1)); + } + + private void scheduleClearDisruptionNow() { + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + } + + private void disconnectOrRestartDataNode() { + if (randomBoolean()) { + disconnectRandomDataNode(); + } else { + testClusterNodes.randomDataNode().ifPresent(TestClusterNode::restart); + } + } + + private void disconnectRandomDataNode() { + testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n)); + } + private void startCluster() { final ClusterState initialClusterState = new ClusterState.Builder(ClusterName.DEFAULT).nodes(testClusterNodes.discoveryNodes()).build(); @@ -329,6 +437,20 @@ private void setupTestCluster(int masterNodes, int dataNodes) { startCluster(); } + private static ActionListener assertNoFailureListener(Consumer consumer) { + return new ActionListener() { + @Override + public void onResponse(final T t) { + consumer.accept(t); + } + + @Override + public void onFailure(final Exception e) { + throw new AssertionError(e); + } + }; + } + private static ActionListener assertNoFailureListener(Runnable r) { return new ActionListener() { @Override @@ -388,6 +510,11 @@ private final class TestClusterNodes { } } + public TestClusterNode nodeById(final String nodeId) { + return nodes.values().stream().filter(n -> n.node.getId().equals(nodeId)).findFirst() + .orElseThrow(() -> new AssertionError("Could not find node by id [" + nodeId + ']')); + } + private TestClusterNode newMasterNode(String nodeName) throws IOException { return newNode(nodeName, DiscoveryNode.Role.MASTER); } @@ -410,9 +537,17 @@ public TestClusterNode randomMasterNode() { ); } - public Optional randomDataNode() { + public Optional randomDataNode(String... excludedNames) { // Select from sorted list of data-nodes here to not have deterministic behaviour final List dataNodes = testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) + .filter(n -> { + for (final String nodeName : excludedNames) { + if (n.node.getName().equals(nodeName)) { + return false; + } + } + return true; + }) .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()); return dataNodes.isEmpty() ? Optional.empty() : Optional.ofNullable(randomFrom(dataNodes)); } @@ -464,7 +599,9 @@ private final class TestClusterNode { private final Logger logger = LogManager.getLogger(TestClusterNode.class); - private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( + Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream() + ).collect(Collectors.toList())); private final TransportService transportService; @@ -671,6 +808,18 @@ allocationService, new AliasValidator(), environment, indexScopedSettings, transportService, clusterService, threadPool, snapshotsService, actionFilters, indexNameExpressionResolver )); + actions.put(ClusterRerouteAction.INSTANCE, + new TransportClusterRerouteAction(transportService, clusterService, threadPool, allocationService, + actionFilters, indexNameExpressionResolver)); + actions.put(ClusterStateAction.INSTANCE, + new TransportClusterStateAction(transportService, clusterService, threadPool, + actionFilters, indexNameExpressionResolver)); + actions.put(IndicesShardStoresAction.INSTANCE, + new TransportIndicesShardStoresAction( + transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, + new TransportNodesListGatewayStartedShards(settings, + threadPool, clusterService, transportService, actionFilters, nodeEnv, indicesService, namedXContentRegistry)) + ); client.initialize(actions, () -> clusterService.localNode().getId(), transportService.getRemoteClusterService()); } @@ -694,6 +843,7 @@ public void restart() { } public void stop() { + testClusterNodes.disconnectNode(this); indicesService.close(); clusterService.close(); indicesClusterStateService.close(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java index b728a1e981313..92698f6537b25 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -48,6 +49,8 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.test.ESTestCase.copyWriteable; import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME; @@ -254,7 +257,9 @@ public String toString() { } private NamedWriteableRegistry writeableRegistry() { - return new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + return new NamedWriteableRegistry( + Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()) + .collect(Collectors.toList())); } public enum ConnectionStatus { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java index ddfcc29c750ce..3c0acca7ff0ae 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java @@ -20,6 +20,7 @@ package org.elasticsearch.test.transport; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; @@ -29,7 +30,10 @@ import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -60,6 +64,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.lucene.util.LuceneTestCase.rarely; @@ -96,7 +102,7 @@ public void handleResponse(final long reque final Response deliveredResponse; try (BytesStreamOutput output = new BytesStreamOutput()) { response.writeTo(output); - deliveredResponse = transportResponseHandler.read(output.bytes().streamInput()); + deliveredResponse = transportResponseHandler.read(new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writeableRegistry())); } catch (IOException | UnsupportedOperationException e) { throw new AssertionError("failed to serialize/deserialize response " + response, e); } @@ -104,6 +110,12 @@ public void handleResponse(final long reque } } + private NamedWriteableRegistry writeableRegistry() { + return new NamedWriteableRegistry( + Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()) + .collect(Collectors.toList())); + } + /** * simulate a local error for the given requestId, will be wrapped * by a {@link SendRequestTransportException} From f9690461b76506a63283434dc36b6c6cb69e6ede Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 15:04:49 +0100 Subject: [PATCH 33/62] bck --- .../snapshots/SnapshotsServiceTests.java | 115 ++++++++++-------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index f2691aff9fadf..6c5f2ec566ae7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -71,7 +70,8 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingService; -import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.elasticsearch.cluster.service.ClusterApplierService; @@ -289,9 +289,9 @@ public void testSnapshotWithNodeDisconnects() { assertThat(snapshotIds, hasSize(1)); } - @Repeat(iterations = 1000) public void testSnapshotPrimaryRelocations() { - setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); + final int masterNodeCount = randomFrom(1, 3, 5); + setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); String repoName = "repo"; String snapshotName = "snapshot"; @@ -315,61 +315,74 @@ public void testSnapshotPrimaryRelocations() { assertNoFailureListener( () -> masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( clusterStateResponse -> { - RoutingTable table = clusterStateResponse.getState().routingTable(); + final ShardRouting shardToRelocate = clusterStateResponse.getState().routingTable().allShards(index).get(0); final TestClusterNode currentPrimaryNode = - testClusterNodes.nodeById(table.allShards(index).get(0).currentNodeId()); + testClusterNodes.nodeById(shardToRelocate.currentNodeId()); final TestClusterNode otherNode = testClusterNodes.randomDataNode(currentPrimaryNode.node.getName()) .orElseThrow(() -> new AssertionError("Could not find another data node.")); - deterministicTaskQueue.scheduleNow(() -> { - currentPrimaryNode.stop(); - Runnable assignStalePrimary = () -> masterAdminClient.cluster().reroute( - new ClusterRerouteRequest().add( - new AllocateEmptyPrimaryAllocationCommand( - index, 0, otherNode.node.getName(), true - ) - ), - new ActionListener() { - @Override - public void onResponse(final ClusterRerouteResponse clusterRerouteResponse) { - masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener( - () -> createdSnapshot.set(true))); - } - - @Override - public void onFailure(final Exception e) { - masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener( - () -> createdSnapshot.set(true))); - } - }); - deterministicTaskQueue.scheduleNow( - new Runnable() { - @Override - public void run() { - if (masterNode.clusterService.state().routingTable().allShards(index).get(0).unassigned() || rarely()) { - assignStalePrimary.run(); + final Runnable maybeForceAllocate = new Runnable() { + @Override + public void run() { + masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( + resp -> { + final ShardRouting shardRouting = resp.getState().routingTable().shardRoutingTable(shardToRelocate.shardId()).primaryShard(); + if (shardRouting.unassigned() + && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { + deterministicTaskQueue.scheduleNow(() -> masterAdminClient.cluster().reroute( + new ClusterRerouteRequest().add( + new AllocateEmptyPrimaryAllocationCommand( + index, shardRouting.shardId().id(), otherNode.node.getName(), true + ) + ), + new ActionListener() { + @Override + public void onResponse(final ClusterRerouteResponse clusterRerouteResponse) { + testClusterNodes.randomMasterNode().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener(() -> { + createdSnapshot.set(true); + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(masterNode)); + deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 200L, + () -> testClusterNodes.clearNetworkDisruptions()); + })); + } + + @Override + public void onFailure(final Exception e) { + testClusterNodes.randomMasterNode().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(assertNoFailureListener(() -> { + createdSnapshot.set(true); + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(masterNode)); + deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 200L, + () -> testClusterNodes.clearNetworkDisruptions()); + })); + } + })); } else { - if (frequently()) { - deterministicTaskQueue.scheduleNow(this); - } else { - deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 10L, this); - } + deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), this); } } - } - ); - }); + )); + } + }; + currentPrimaryNode.stop(); + testClusterNodes.nodes.remove(currentPrimaryNode.node.getName()); + deterministicTaskQueue.scheduleNow(maybeForceAllocate); } )))))); runUntil(() -> { - final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); - return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); + final SnapshotsInProgress snapshotsInProgress = testClusterNodes.randomMasterNode().clusterService.state().custom(SnapshotsInProgress.TYPE); + return (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) && createdSnapshot.get(); + }, TimeUnit.MINUTES.toMillis(20L)); + + testClusterNodes.clearNetworkDisruptions(); + runUntil(() -> { + final List versions = testClusterNodes.nodes.values().stream().map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); + return versions.size() == 1L; }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); - SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNode().orElseThrow(() -> new AssertionError("No data nodes running")).clusterService.state().custom(SnapshotsInProgress.TYPE); assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = masterNode.repositoriesService.repository(repoName); Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); @@ -561,11 +574,13 @@ public void disconnectNode(TestClusterNode node) { } public void clearNetworkDisruptions() { - disruptedLinks.clear(); disruptedLinks.disconnected.forEach(nodeName -> { - final DiscoveryNode node = testClusterNodes.nodes.get(nodeName).node; - testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().openConnection(node, null)); + if (testClusterNodes.nodes.containsKey(nodeName)) { + final DiscoveryNode node = testClusterNodes.nodes.get(nodeName).node; + testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().openConnection(node, null)); + } }); + disruptedLinks.clear(); } private NetworkDisruption.DisruptedLinks getDisruption() { @@ -832,7 +847,7 @@ public void restart() { try { final TestClusterNode restartedNode = new TestClusterNode( - new DiscoveryNode(node.getName(), node.getId(), buildNewFakeTransportAddress(), emptyMap(), + new DiscoveryNode(node.getName(), node.getId(), node.getAddress(), emptyMap(), node.getRoles(), Version.CURRENT), disruption); testClusterNodes.nodes.put(node.getName(), restartedNode); restartedNode.start(oldState); From cea40ea9dbc02f243f9ff7a86ef2b7cad53ec197 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 16:50:10 +0100 Subject: [PATCH 34/62] reproduced --- .../snapshots/SnapshotsServiceTests.java | 67 +++++++++++-------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 7be46988a2bae..453d0376d5baa 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.snapshots; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -29,7 +30,6 @@ import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteAction; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; @@ -48,6 +48,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterModule; @@ -152,6 +153,7 @@ import static org.elasticsearch.env.Environment.PATH_HOME_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; @@ -341,6 +343,8 @@ public void testConcurrentSnapshotCreateAndDelete() { assertEquals(0, snapshotInfo.failedShards()); } + // -Dtests.seed=92C2A9BD03C14003 ... 35 runs + @Repeat(iterations = 1000) public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); @@ -380,35 +384,36 @@ public void run() { final ShardRouting shardRouting = resp.getState().routingTable().shardRoutingTable(shardToRelocate.shardId()).primaryShard(); if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { + if (masterNodeCount > 1) { + deterministicTaskQueue.scheduleNow(()->{ + masterNode.stop(); + testClusterNodes.nodes.remove(masterNode.node.getName()); + }); + } + testClusterNodes.randomDataNode().get().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .execute(ActionListener.wrap(() -> { + testClusterNodes.randomDataNode().get().client.admin().cluster().deleteSnapshot( + new DeleteSnapshotRequest(repoName, snapshotName), new ActionListener() { + @Override + public void onResponse(final AcknowledgedResponse acknowledgedResponse) { + + } + + @Override + public void onFailure(final Exception e) { + + } + }); + createdSnapshot.set(true); + })); deterministicTaskQueue.scheduleNow(() -> masterAdminClient.cluster().reroute( new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true ) ), - new ActionListener() { - @Override - public void onResponse(final ClusterRerouteResponse clusterRerouteResponse) { - testClusterNodes.randomMasterNode().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener(() -> { - createdSnapshot.set(true); - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(masterNode)); - deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 200L, - () -> testClusterNodes.clearNetworkDisruptions()); - })); - } - - @Override - public void onFailure(final Exception e) { - testClusterNodes.randomMasterNode().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) - .execute(assertNoFailureListener(() -> { - createdSnapshot.set(true); - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.disconnectNode(masterNode)); - deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + 200L, - () -> testClusterNodes.clearNetworkDisruptions()); - })); - } - })); + ActionListener.wrap(() -> { + }))); } else { deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), this); } @@ -416,8 +421,10 @@ public void onFailure(final Exception e) { )); } }; - currentPrimaryNode.stop(); - testClusterNodes.nodes.remove(currentPrimaryNode.node.getName()); + deterministicTaskQueue.scheduleNow(() -> { + currentPrimaryNode.stop(); + testClusterNodes.nodes.remove(currentPrimaryNode.node.getName()); + }); deterministicTaskQueue.scheduleNow(maybeForceAllocate); } )))))); @@ -438,7 +445,7 @@ public void onFailure(final Exception e) { assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = masterNode.repositoriesService.repository(repoName); Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); - assertThat(snapshotIds, hasSize(1)); + assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0))); } private void scheduleClearDisruptionNow() { @@ -621,8 +628,8 @@ public void disconnectNode(TestClusterNode node) { if (disruptedLinks.disconnected.contains(node.node.getName())) { return; } - disruptedLinks.disconnect(node.node.getName()); testClusterNodes.nodes.values().forEach(n -> n.transportService.getConnectionManager().disconnectFromNode(node.node)); + disruptedLinks.disconnect(node.node.getName()); } public void clearNetworkDisruptions() { @@ -723,6 +730,10 @@ protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { if (node.getName().equals(destination.getName())) { return ConnectionStatus.CONNECTED; } + if (testClusterNodes.nodes.containsKey(node.getName()) == false + || testClusterNodes.nodes.containsKey(destination.getName()) == false) { + return ConnectionStatus.DISCONNECTED; + } return disruption.get().disrupt(node.getName(), destination.getName()) ? ConnectionStatus.DISCONNECTED : ConnectionStatus.CONNECTED; } From 86b1f32775e44f1a3c77341d67ebe1a06c6c1399 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 18:00:53 +0100 Subject: [PATCH 35/62] fix 1 --- .../snapshots/SnapshotShardsService.java | 18 +++++++++++++++--- .../snapshots/SnapshotsServiceTests.java | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 6b7b506114361..632ebecf5f643 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -72,6 +72,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; @@ -272,12 +273,10 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { // Abort all running shards for this snapshot Map snapshotShards = shardSnapshots.get(entry.snapshot()); if (snapshotShards != null) { - final String failure = "snapshot has been aborted"; for (ObjectObjectCursor shard : entry.shards()) { - final IndexShardSnapshotStatus snapshotStatus = snapshotShards.get(shard.key); if (snapshotStatus != null) { - final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.abortIfNotCompleted(failure); + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.abortIfNotCompleted("snapshot has been aborted"); final Stage stage = lastSnapshotStatus.getStage(); if (stage == Stage.FINALIZE) { logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, " + @@ -295,6 +294,19 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } } + } else { + final ImmutableOpenMap clusterStateShards = entry.shards(); + if (clusterStateShards.isEmpty()) { + + } else { + for (ObjectObjectCursor curr : clusterStateShards) { + if (curr.value.state() == State.ABORTED) { + // due to CS batching we might have missed the INIT state and straight went into ABORTED + // notify master that abort has completed by moving to FAILED + notifyFailedSnapshotShard(entry.snapshot(), curr.key, localNodeId, curr.value.reason()); + } + } + } } } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 453d0376d5baa..368c3a9436f2b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -344,6 +344,8 @@ public void testConcurrentSnapshotCreateAndDelete() { } // -Dtests.seed=92C2A9BD03C14003 ... 35 runs + // -ea -Dtests.seed=92C2A9BD03C14003:AE482EC68DB8B206 + // 262C019B8568F4DF:6C6B6D998E1745E6 @Repeat(iterations = 1000) public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); From a8b20eb7468a2a3a92678e3a1dac4105fd4bf4f6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 18:00:56 +0100 Subject: [PATCH 36/62] Revert "Revert "Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its"""" This reverts commit 1c0a3b98a97f0a26da5362eb2e9a1f83cfe00c18. --- .../routing/RoutingChangesObserver.java | 49 ++++++ .../routing/allocation/AllocationService.java | 19 ++- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ++++++++++++++++++ 5 files changed, 219 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 883b4c22f7fc0..0c0d3c5a099f9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,6 +19,8 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.index.shard.ShardId; + /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -211,4 +213,51 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } + + abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { + + @Override + public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { + onChanged(unassignedShard.shardId()); + } + + @Override + public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { + onChanged(initializingShard.shardId()); + } + @Override + public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { + onChanged(startedShard.shardId()); + } + @Override + public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { + onChanged(unassignedShard.shardId()); + } + @Override + public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { + onChanged(failedShard.shardId()); + } + @Override + public void relocationCompleted(ShardRouting removedRelocationSource) { + onChanged(removedRelocationSource.shardId()); + } + @Override + public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { + onChanged(removedReplicaRelocationSource.shardId()); + } + @Override + public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { + onChanged(startedPrimaryShard.shardId()); + } + @Override + public void replicaPromoted(ShardRouting replicaShard) { + onChanged(replicaShard.shardId()); + } + @Override + public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { + onChanged(oldReplica.shardId()); + } + + protected abstract void onChanged(ShardId shardId); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 59f43a193ddc8..389eec6ed5a87 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -136,15 +137,29 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); + ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); - newStateBuilder.customs(customsBuilder.build()); } } + final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + SnapshotsInProgress updatedSnapshotsInProgress = + allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); + if (updatedSnapshotsInProgress != snapshotsInProgress) { + if (customsBuilder == null) { + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + } + customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); + } + } + if (customsBuilder != null) { + newStateBuilder.customs(customsBuilder.build()); + } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index e0be712a230c7..f178ca18845ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -76,11 +78,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); + private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater ); - /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -251,6 +253,14 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } + /** + * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes + */ + public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, + RoutingTable newRoutingTable) { + return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); + } + /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 632ebecf5f643..23fbc2d8d576b 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -325,7 +325,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to starts + // We have new shards to start if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 4e8c26ea593d6..30def22893dfd 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -45,6 +46,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -676,12 +678,28 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); + // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here + assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } + private boolean assertConsistency(ClusterState state) { + SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + assert snapshotsInProgress == updateWithRoutingTable( + snapshotsInProgress.entries().stream().flatMap(entry -> { + Iterable iterable = () -> entry.shards().keysIt(); + return StreamSupport.stream(iterable.spliterator(), false); + }).collect(Collectors.toSet()), + snapshotsInProgress, state.routingTable() + ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; + } + return true; + } + /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1582,4 +1600,126 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } + + public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { + + private final Set shardChanges = new HashSet<>(); + + public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { + return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); + } + + @Override + protected void onChanged(ShardId shardId) { + shardChanges.add(shardId); + } + } + + private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, + RoutingTable newRoutingTable) { + if (oldSnapshot == null || shardIds.isEmpty()) { + return oldSnapshot; + } + List entries = new ArrayList<>(); + boolean snapshotsInProgressChanged = false; + for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { + ImmutableOpenMap.Builder shardsBuilder = null; + for (ShardId shardId : shardIds) { + final ImmutableOpenMap shards = entry.shards(); + final ShardSnapshotStatus currentStatus = shards.get(shardId); + if (currentStatus != null && currentStatus.state().completed() == false) { + IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); + assert routingTable != null; + final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) + .map(IndexShardRoutingTable::primaryShard) + .map( + primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) + ) + .orElse(failedStatus(null, "missing shard")); + if (newStatus != currentStatus) { + if (shardsBuilder == null) { + shardsBuilder = ImmutableOpenMap.builder(shards); + } + shardsBuilder.put(shardId, newStatus); + } + } + } + if (shardsBuilder == null) { + entries.add(entry); + } else { + snapshotsInProgressChanged = true; + ImmutableOpenMap shards = shardsBuilder.build(); + entries.add( + new SnapshotsInProgress.Entry( + entry, + completed(shards.values()) ? State.SUCCESS : entry.state(), + shards + ) + ); + } + } + if (snapshotsInProgressChanged) { + return new SnapshotsInProgress(entries); + } + return oldSnapshot; + } + + private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, + final ShardRouting primaryShardRouting) { + final State currentState = currentStatus.state(); + final ShardSnapshotStatus newStatus; + if (primaryShardRouting.active() == false) { + if (primaryShardRouting.initializing() && currentState == State.WAITING) { + newStatus = currentStatus; + } else { + newStatus = failedStatus( + primaryShardRouting.currentNodeId(), + primaryShardRouting.unassignedInfo().getReason().toString() + ); + } + } else if (primaryShardRouting.started()) { + switch (currentState) { + case WAITING: + newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); + break; + case INIT: { + String currentNodeId = currentStatus.nodeId(); + assert currentNodeId != null; + if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + } + case ABORTED: + String currentNodeId = currentStatus.nodeId(); + if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + default: + newStatus = currentStatus; + break; + } + } else { + assert primaryShardRouting.relocating(); + if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { + newStatus = failedStatus(currentStatus.nodeId()); + } else { + newStatus = currentStatus; + } + } + return newStatus; + } + + private static ShardSnapshotStatus failedStatus(String nodeId) { + return failedStatus(nodeId, "shard failed"); + } + + private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { + return new ShardSnapshotStatus(nodeId, State.FAILED, reason); + } } From c3836f3afd75964952392117233cd362aa02c8d2 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 18:18:45 +0100 Subject: [PATCH 37/62] bck --- .../java/org/elasticsearch/snapshots/SnapshotsServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 368c3a9436f2b..6e51922bd11fa 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -346,7 +346,7 @@ public void testConcurrentSnapshotCreateAndDelete() { // -Dtests.seed=92C2A9BD03C14003 ... 35 runs // -ea -Dtests.seed=92C2A9BD03C14003:AE482EC68DB8B206 // 262C019B8568F4DF:6C6B6D998E1745E6 - @Repeat(iterations = 1000) + @Repeat(iterations = 100) public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); From a3bc9fc5b9e7852273e95d02b593b31b8f05042a Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 18:18:58 +0100 Subject: [PATCH 38/62] Revert "Revert "Revert "Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its""""" This reverts commit a8b20eb7468a2a3a92678e3a1dac4105fd4bf4f6. --- .../routing/RoutingChangesObserver.java | 49 ------ .../routing/allocation/AllocationService.java | 19 +-- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ------------------ 5 files changed, 5 insertions(+), 219 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 0c0d3c5a099f9..883b4c22f7fc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.index.shard.ShardId; - /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -213,51 +211,4 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } - - abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { - - @Override - public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { - onChanged(unassignedShard.shardId()); - } - - @Override - public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { - onChanged(initializingShard.shardId()); - } - @Override - public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { - onChanged(startedShard.shardId()); - } - @Override - public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { - onChanged(unassignedShard.shardId()); - } - @Override - public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { - onChanged(failedShard.shardId()); - } - @Override - public void relocationCompleted(ShardRouting removedRelocationSource) { - onChanged(removedRelocationSource.shardId()); - } - @Override - public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { - onChanged(removedReplicaRelocationSource.shardId()); - } - @Override - public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { - onChanged(startedPrimaryShard.shardId()); - } - @Override - public void replicaPromoted(ShardRouting replicaShard) { - onChanged(replicaShard.shardId()); - } - @Override - public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { - onChanged(oldReplica.shardId()); - } - - protected abstract void onChanged(ShardId shardId); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 389eec6ed5a87..59f43a193ddc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -137,29 +136,15 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); - ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); + newStateBuilder.customs(customsBuilder.build()); } } - final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - SnapshotsInProgress updatedSnapshotsInProgress = - allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); - if (updatedSnapshotsInProgress != snapshotsInProgress) { - if (customsBuilder == null) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); - } - customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); - } - } - if (customsBuilder != null) { - newStateBuilder.customs(customsBuilder.build()); - } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index f178ca18845ae..e0be712a230c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -39,7 +38,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -78,11 +76,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); - private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater ); + /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -253,14 +251,6 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } - /** - * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes - */ - public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, - RoutingTable newRoutingTable) { - return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); - } - /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 23fbc2d8d576b..632ebecf5f643 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -325,7 +325,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to start + // We have new shards to starts if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 30def22893dfd..4e8c26ea593d6 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -46,7 +45,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -678,28 +676,12 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); - // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here - assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } - private boolean assertConsistency(ClusterState state) { - SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - assert snapshotsInProgress == updateWithRoutingTable( - snapshotsInProgress.entries().stream().flatMap(entry -> { - Iterable iterable = () -> entry.shards().keysIt(); - return StreamSupport.stream(iterable.spliterator(), false); - }).collect(Collectors.toSet()), - snapshotsInProgress, state.routingTable() - ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; - } - return true; - } - /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1600,126 +1582,4 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } - - public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { - - private final Set shardChanges = new HashSet<>(); - - public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { - return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); - } - - @Override - protected void onChanged(ShardId shardId) { - shardChanges.add(shardId); - } - } - - private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, - RoutingTable newRoutingTable) { - if (oldSnapshot == null || shardIds.isEmpty()) { - return oldSnapshot; - } - List entries = new ArrayList<>(); - boolean snapshotsInProgressChanged = false; - for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { - ImmutableOpenMap.Builder shardsBuilder = null; - for (ShardId shardId : shardIds) { - final ImmutableOpenMap shards = entry.shards(); - final ShardSnapshotStatus currentStatus = shards.get(shardId); - if (currentStatus != null && currentStatus.state().completed() == false) { - IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); - assert routingTable != null; - final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) - .map(IndexShardRoutingTable::primaryShard) - .map( - primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) - ) - .orElse(failedStatus(null, "missing shard")); - if (newStatus != currentStatus) { - if (shardsBuilder == null) { - shardsBuilder = ImmutableOpenMap.builder(shards); - } - shardsBuilder.put(shardId, newStatus); - } - } - } - if (shardsBuilder == null) { - entries.add(entry); - } else { - snapshotsInProgressChanged = true; - ImmutableOpenMap shards = shardsBuilder.build(); - entries.add( - new SnapshotsInProgress.Entry( - entry, - completed(shards.values()) ? State.SUCCESS : entry.state(), - shards - ) - ); - } - } - if (snapshotsInProgressChanged) { - return new SnapshotsInProgress(entries); - } - return oldSnapshot; - } - - private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, - final ShardRouting primaryShardRouting) { - final State currentState = currentStatus.state(); - final ShardSnapshotStatus newStatus; - if (primaryShardRouting.active() == false) { - if (primaryShardRouting.initializing() && currentState == State.WAITING) { - newStatus = currentStatus; - } else { - newStatus = failedStatus( - primaryShardRouting.currentNodeId(), - primaryShardRouting.unassignedInfo().getReason().toString() - ); - } - } else if (primaryShardRouting.started()) { - switch (currentState) { - case WAITING: - newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); - break; - case INIT: { - String currentNodeId = currentStatus.nodeId(); - assert currentNodeId != null; - if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - } - case ABORTED: - String currentNodeId = currentStatus.nodeId(); - if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - default: - newStatus = currentStatus; - break; - } - } else { - assert primaryShardRouting.relocating(); - if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { - newStatus = failedStatus(currentStatus.nodeId()); - } else { - newStatus = currentStatus; - } - } - return newStatus; - } - - private static ShardSnapshotStatus failedStatus(String nodeId) { - return failedStatus(nodeId, "shard failed"); - } - - private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { - return new ShardSnapshotStatus(nodeId, State.FAILED, reason); - } } From b0d4c9945b1b27fdafdf5a7f9601f3ece697f0bd Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 20:28:28 +0100 Subject: [PATCH 39/62] bck --- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 6e51922bd11fa..7f27ac93f202d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.snapshots; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -346,7 +345,8 @@ public void testConcurrentSnapshotCreateAndDelete() { // -Dtests.seed=92C2A9BD03C14003 ... 35 runs // -ea -Dtests.seed=92C2A9BD03C14003:AE482EC68DB8B206 // 262C019B8568F4DF:6C6B6D998E1745E6 - @Repeat(iterations = 100) + // ./gradlew null -Dtests.seed=C5E2A2FA6692D387 -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotPrimaryRelocations [seed=[C5E2A2FA6692D387:3EB447C9FE5056A]]" -Dtests.locale=vun-TZ -Dtests.timezone=Etc/GMT+10 + // ./gradlew :server:unitTest -Dtests.seed=1163EDA5C788BA47 -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotPrimaryRelocations" -Dtests.security.manager=true -Dtests.locale=yo -Dtests.timezone=Etc/GMT-1 -Dcompiler.java=11 -Druntime.java=11 public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); From db2fd56434b3b399b6953da321b35a2e7022c1d1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 23:21:02 +0100 Subject: [PATCH 40/62] tests pass --- .../snapshots/SnapshotShardsService.java | 4 +- .../snapshots/SnapshotsService.java | 6 +- .../snapshots/SnapshotsServiceTests.java | 70 ++++++++++--------- .../test/transport/MockTransport.java | 3 +- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 632ebecf5f643..290589a5649a7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -72,7 +72,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; @@ -276,7 +275,8 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { for (ObjectObjectCursor shard : entry.shards()) { final IndexShardSnapshotStatus snapshotStatus = snapshotShards.get(shard.key); if (snapshotStatus != null) { - final IndexShardSnapshotStatus.Copy lastSnapshotStatus = snapshotStatus.abortIfNotCompleted("snapshot has been aborted"); + final IndexShardSnapshotStatus.Copy lastSnapshotStatus = + snapshotStatus.abortIfNotCompleted("snapshot has been aborted"); final Stage stage = lastSnapshotStatus.getStage(); if (stage == Stage.FINALIZE) { logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, " + diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 4e8c26ea593d6..9b788f2ce0447 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1197,7 +1197,11 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (state == State.INIT) { // snapshot is still initializing, mark it as aborted shards = snapshotEntry.shards(); - + if (shards.isEmpty()) { + // No shards in this snapshot, we delete it right away since the SnapshotShardsService + // has no work to do. + endSnapshot(snapshotEntry); + } } else if (state == State.STARTED) { // snapshot is started - mark every non completed shard as aborted final ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 7f27ac93f202d..b87729179dbc5 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -32,12 +32,12 @@ import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.TransportDeleteSnapshotAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; @@ -47,7 +47,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.TransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterModule; @@ -342,11 +341,6 @@ public void testConcurrentSnapshotCreateAndDelete() { assertEquals(0, snapshotInfo.failedShards()); } - // -Dtests.seed=92C2A9BD03C14003 ... 35 runs - // -ea -Dtests.seed=92C2A9BD03C14003:AE482EC68DB8B206 - // 262C019B8568F4DF:6C6B6D998E1745E6 - // ./gradlew null -Dtests.seed=C5E2A2FA6692D387 -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotPrimaryRelocations [seed=[C5E2A2FA6692D387:3EB447C9FE5056A]]" -Dtests.locale=vun-TZ -Dtests.timezone=Etc/GMT+10 - // ./gradlew :server:unitTest -Dtests.seed=1163EDA5C788BA47 -Dtests.class=org.elasticsearch.snapshots.SnapshotsServiceTests -Dtests.method="testSnapshotPrimaryRelocations" -Dtests.security.manager=true -Dtests.locale=yo -Dtests.timezone=Etc/GMT-1 -Dcompiler.java=11 -Druntime.java=11 public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); @@ -373,7 +367,8 @@ public void testSnapshotPrimaryRelocations() { assertNoFailureListener( () -> masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( clusterStateResponse -> { - final ShardRouting shardToRelocate = clusterStateResponse.getState().routingTable().allShards(index).get(0); + final ShardRouting shardToRelocate = + clusterStateResponse.getState().routingTable().allShards(index).get(0); final TestClusterNode currentPrimaryNode = testClusterNodes.nodeById(shardToRelocate.currentNodeId()); final TestClusterNode otherNode = testClusterNodes.randomDataNode(currentPrimaryNode.node.getName()) @@ -383,29 +378,22 @@ public void testSnapshotPrimaryRelocations() { public void run() { masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( resp -> { - final ShardRouting shardRouting = resp.getState().routingTable().shardRoutingTable(shardToRelocate.shardId()).primaryShard(); + final ShardRouting shardRouting = resp.getState().routingTable() + .shardRoutingTable(shardToRelocate.shardId()).primaryShard(); if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { if (masterNodeCount > 1) { - deterministicTaskQueue.scheduleNow(()->{ + deterministicTaskQueue.scheduleNow(() -> { masterNode.stop(); testClusterNodes.nodes.remove(masterNode.node.getName()); }); } - testClusterNodes.randomDataNode().get().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + testClusterNodes.randomDataNode().get().client.admin().cluster() + .prepareCreateSnapshot(repoName, snapshotName) .execute(ActionListener.wrap(() -> { - testClusterNodes.randomDataNode().get().client.admin().cluster().deleteSnapshot( - new DeleteSnapshotRequest(repoName, snapshotName), new ActionListener() { - @Override - public void onResponse(final AcknowledgedResponse acknowledgedResponse) { - - } - - @Override - public void onFailure(final Exception e) { - - } - }); + testClusterNodes.randomDataNode().get().client.admin() + .cluster().deleteSnapshot( + new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); createdSnapshot.set(true); })); deterministicTaskQueue.scheduleNow(() -> masterAdminClient.cluster().reroute( @@ -413,11 +401,12 @@ public void onFailure(final Exception e) { new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true ) - ), - ActionListener.wrap(() -> { - }))); + ), noopListener())); } else { - deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), this); + deterministicTaskQueue.scheduleAt( + deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), + this + ); } } )); @@ -432,24 +421,40 @@ public void onFailure(final Exception e) { )))))); runUntil(() -> { - final SnapshotsInProgress snapshotsInProgress = testClusterNodes.randomMasterNode().clusterService.state().custom(SnapshotsInProgress.TYPE); + final SnapshotsInProgress snapshotsInProgress = + testClusterNodes.randomMasterNode().clusterService.state().custom(SnapshotsInProgress.TYPE); return (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) && createdSnapshot.get(); }, TimeUnit.MINUTES.toMillis(20L)); testClusterNodes.clearNetworkDisruptions(); runUntil(() -> { - final List versions = testClusterNodes.nodes.values().stream().map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); + final List versions = testClusterNodes.nodes.values().stream() + .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); return versions.size() == 1L; }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); - SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNode().orElseThrow(() -> new AssertionError("No data nodes running")).clusterService.state().custom(SnapshotsInProgress.TYPE); + SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNode() + .orElseThrow(() -> new AssertionError("No data nodes running")) + .clusterService.state().custom(SnapshotsInProgress.TYPE); assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = masterNode.repositoriesService.repository(repoName); Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0))); } + private static ActionListener noopListener() { + return new ActionListener() { + @Override + public void onResponse(final T t) { + } + + @Override + public void onFailure(final Exception e) { + } + }; + } + private void scheduleClearDisruptionNow() { deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); } @@ -758,6 +763,7 @@ protected void execute(Runnable runnable) { @Override public TransportRequestHandler interceptHandler(String action, String executor, boolean forceExecution, TransportRequestHandler actualHandler) { + // TODO: Remove this hack once recoveries are async and can be used in these tests if (action.startsWith("internal:index/shard/recovery")) { return (request, channel, task) -> deterministicTaskQueue.scheduleAt( deterministicTaskQueue.getCurrentTimeMillis() + 20L, diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java index 3c0acca7ff0ae..c100dbba4e117 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java @@ -102,7 +102,8 @@ public void handleResponse(final long reque final Response deliveredResponse; try (BytesStreamOutput output = new BytesStreamOutput()) { response.writeTo(output); - deliveredResponse = transportResponseHandler.read(new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writeableRegistry())); + deliveredResponse = transportResponseHandler.read( + new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writeableRegistry())); } catch (IOException | UnsupportedOperationException e) { throw new AssertionError("failed to serialize/deserialize response " + response, e); } From cc902e43e4bdd5479ba2b9c029f3d6251127bd83 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 23:27:47 +0100 Subject: [PATCH 41/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index b87729179dbc5..0af869908b57d 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -279,9 +279,6 @@ public void testSnapshotWithNodeDisconnects() { })))); runUntil(() -> { - if (createdSnapshot.get() == false) { - return false; - } final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); }, TimeUnit.MINUTES.toMillis(20L)); @@ -391,17 +388,18 @@ public void run() { testClusterNodes.randomDataNode().get().client.admin().cluster() .prepareCreateSnapshot(repoName, snapshotName) .execute(ActionListener.wrap(() -> { - testClusterNodes.randomDataNode().get().client.admin() - .cluster().deleteSnapshot( + testClusterNodes.randomDataNode().orElseThrow( + () -> new AssertionError("Expected at least one active data node") + ).client.admin().cluster().deleteSnapshot( new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); createdSnapshot.set(true); })); - deterministicTaskQueue.scheduleNow(() -> masterAdminClient.cluster().reroute( - new ClusterRerouteRequest().add( - new AllocateEmptyPrimaryAllocationCommand( - index, shardRouting.shardId().id(), otherNode.node.getName(), true - ) - ), noopListener())); + deterministicTaskQueue.scheduleNow( + () -> masterAdminClient.cluster().reroute( + new ClusterRerouteRequest().add( + new AllocateEmptyPrimaryAllocationCommand( + index, shardRouting.shardId().id(), otherNode.node.getName(), true) + ), noopListener())); } else { deterministicTaskQueue.scheduleAt( deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), @@ -443,18 +441,6 @@ public void run() { assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0))); } - private static ActionListener noopListener() { - return new ActionListener() { - @Override - public void onResponse(final T t) { - } - - @Override - public void onFailure(final Exception e) { - } - }; - } - private void scheduleClearDisruptionNow() { deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); } @@ -544,6 +530,18 @@ public void onFailure(final Exception e) { }; } + private static ActionListener noopListener() { + return new ActionListener() { + @Override + public void onResponse(final T t) { + } + + @Override + public void onFailure(final Exception e) { + } + }; + } + /** * Create a {@link Environment} with random path.home and path.repo **/ From 54ad3df2382453318b9bcfeefb46a971697854b1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 23:35:26 +0100 Subject: [PATCH 42/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 7 +------ .../test/disruption/DisruptableMockTransport.java | 2 +- .../test/transport/MockTransport.java | 14 ++------------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 0af869908b57d..bb6ee9e9228cd 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -49,7 +49,6 @@ import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ESAllocationTestCase; @@ -80,7 +79,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -144,7 +142,6 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -678,9 +675,7 @@ private final class TestClusterNode { private final Logger logger = LogManager.getLogger(TestClusterNode.class); - private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( - Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream() - ).collect(Collectors.toList())); + private final NamedWriteableRegistry namedWriteableRegistry = DisruptableMockTransport.writeableRegistry(); private final TransportService transportService; diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java index 92698f6537b25..612ff8ab09bf2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java @@ -256,7 +256,7 @@ public String toString() { } } - private NamedWriteableRegistry writeableRegistry() { + public static NamedWriteableRegistry writeableRegistry() { return new NamedWriteableRegistry( Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()) .collect(Collectors.toList())); diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java index c100dbba4e117..ae61ca1e192a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.transport; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; @@ -31,13 +30,12 @@ import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.CloseableConnection; import org.elasticsearch.transport.ConnectionManager; @@ -64,8 +62,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.apache.lucene.util.LuceneTestCase.rarely; @@ -103,7 +99,7 @@ public void handleResponse(final long reque try (BytesStreamOutput output = new BytesStreamOutput()) { response.writeTo(output); deliveredResponse = transportResponseHandler.read( - new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writeableRegistry())); + new NamedWriteableAwareStreamInput(output.bytes().streamInput(), DisruptableMockTransport.writeableRegistry())); } catch (IOException | UnsupportedOperationException e) { throw new AssertionError("failed to serialize/deserialize response " + response, e); } @@ -111,12 +107,6 @@ public void handleResponse(final long reque } } - private NamedWriteableRegistry writeableRegistry() { - return new NamedWriteableRegistry( - Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()) - .collect(Collectors.toList())); - } - /** * simulate a local error for the given requestId, will be wrapped * by a {@link SendRequestTransportException} From 592d7cae4373ba591ace05ec07a5dcb9116e2aa7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 21 Jan 2019 23:52:37 +0100 Subject: [PATCH 43/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 81 ++++++++++--------- .../MockSinglePrioritizingExecutor.java | 1 + 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index bb6ee9e9228cd..1861530349752 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -190,10 +190,8 @@ public void testSuccessfulSnapshot() { .execute( assertNoFailureListener( () -> masterNode.client.admin().indices().create( - new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( - Settings.builder() - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL) + .settings(defaultIndexSettings(shards)), assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> createdSnapshot.set(true))))))); @@ -235,10 +233,8 @@ public void testSnapshotWithNodeDisconnects() { assertNoFailureListener( () -> masterNode.client.admin().indices().create( - new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( - Settings.builder() - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL) + .settings(defaultIndexSettings(shards)), assertNoFailureListener( () -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { @@ -254,14 +250,7 @@ public void testSnapshotWithNodeDisconnects() { } final boolean disconnectedMaster = randomBoolean(); if (disconnectedMaster) { - deterministicTaskQueue.scheduleNow( - () -> { - if (randomBoolean()) { - testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode()); - } else { - testClusterNodes.randomDataNode().ifPresent(TestClusterNode::restart); - } - }); + deterministicTaskQueue.scheduleNow(this::disconnectOrRestartMasterNode); } if (disconnectedMaster || randomBoolean()) { deterministicTaskQueue.scheduleAt( @@ -305,10 +294,8 @@ public void testConcurrentSnapshotCreateAndDelete() { .execute( assertNoFailureListener( () -> masterNode.client.admin().indices().create( - new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( - Settings.builder() - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL) + .settings(defaultIndexSettings(shards)), assertNoFailureListener( () -> masterNode.client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener( @@ -354,10 +341,8 @@ public void testSnapshotPrimaryRelocations() { .execute( assertNoFailureListener( () -> masterAdminClient.indices().create( - new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL).settings( - Settings.builder() - .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) - .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)), + new CreateIndexRequest(index).waitForActiveShards(ActiveShardCount.ALL) + .settings(defaultIndexSettings(shards)), assertNoFailureListener( () -> masterAdminClient.cluster().state(new ClusterStateRequest(), assertNoFailureListener( clusterStateResponse -> { @@ -365,8 +350,8 @@ public void testSnapshotPrimaryRelocations() { clusterStateResponse.getState().routingTable().allShards(index).get(0); final TestClusterNode currentPrimaryNode = testClusterNodes.nodeById(shardToRelocate.currentNodeId()); - final TestClusterNode otherNode = testClusterNodes.randomDataNode(currentPrimaryNode.node.getName()) - .orElseThrow(() -> new AssertionError("Could not find another data node.")); + final TestClusterNode otherNode = + testClusterNodes.randomDataNodeSafe(currentPrimaryNode.node.getName()); final Runnable maybeForceAllocate = new Runnable() { @Override public void run() { @@ -377,18 +362,15 @@ public void run() { if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { if (masterNodeCount > 1) { - deterministicTaskQueue.scheduleNow(() -> { - masterNode.stop(); - testClusterNodes.nodes.remove(masterNode.node.getName()); - }); + deterministicTaskQueue.scheduleNow( + () -> testClusterNodes.stopNode(masterNode)); } - testClusterNodes.randomDataNode().get().client.admin().cluster() + testClusterNodes.randomDataNodeSafe().client.admin().cluster() .prepareCreateSnapshot(repoName, snapshotName) .execute(ActionListener.wrap(() -> { - testClusterNodes.randomDataNode().orElseThrow( - () -> new AssertionError("Expected at least one active data node") - ).client.admin().cluster().deleteSnapshot( - new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); + testClusterNodes.randomDataNodeSafe().client.admin().cluster() + .deleteSnapshot( + new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); createdSnapshot.set(true); })); deterministicTaskQueue.scheduleNow( @@ -407,10 +389,7 @@ public void run() { )); } }; - deterministicTaskQueue.scheduleNow(() -> { - currentPrimaryNode.stop(); - testClusterNodes.nodes.remove(currentPrimaryNode.node.getName()); - }); + deterministicTaskQueue.scheduleNow(() -> testClusterNodes.stopNode(currentPrimaryNode)); deterministicTaskQueue.scheduleNow(maybeForceAllocate); } )))))); @@ -450,6 +429,14 @@ private void disconnectOrRestartDataNode() { } } + private void disconnectOrRestartMasterNode() { + if (randomBoolean()) { + testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode()); + } else { + testClusterNodes.randomMasterNode().restart(); + } + } + private void disconnectRandomDataNode() { testClusterNodes.randomDataNode().ifPresent(n -> testClusterNodes.disconnectNode(n)); } @@ -499,6 +486,13 @@ private void setupTestCluster(int masterNodes, int dataNodes) { startCluster(); } + private static Settings defaultIndexSettings(int shards) { + // TODO: randomize replica count settings once recovery operations aren't blocking anymore + return Settings.builder() + .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shards) + .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0).build(); + } + private static ActionListener assertNoFailureListener(Consumer consumer) { return new ActionListener() { @Override @@ -611,6 +605,15 @@ public TestClusterNode randomMasterNode() { ); } + public void stopNode(TestClusterNode node) { + node.stop(); + nodes.remove(node.node.getName()); + } + + public TestClusterNode randomDataNodeSafe(String... excludedNames) { + return randomDataNode(excludedNames).orElseThrow(() -> new AssertionError("Could not find another data node.")); + } + public Optional randomDataNode(String... excludedNames) { // Select from sorted list of data-nodes here to not have deterministic behaviour final List dataNodes = testClusterNodes.nodes.values().stream().filter(n -> n.node.isDataNode()) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 7e37871a2bcb0..032c719b4997a 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -53,6 +53,7 @@ protected void afterExecute(Runnable r, Throwable t) { throw new KillWorkerError(); } + // Override awaitTermination to not get blocked on termination condition @Override public boolean awaitTermination(long timeout, TimeUnit unit) { return true; From 7173c808f65724ea0c0f7f8b3127a6f22f2c04b1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 00:07:37 +0100 Subject: [PATCH 44/62] another reproducer --- .../snapshots/SnapshotsServiceTests.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 1861530349752..7cc03cc6f7d83 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -238,7 +238,7 @@ public void testSnapshotWithNodeDisconnects() { assertNoFailureListener( () -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { - deterministicTaskQueue.scheduleNow(this::disconnectRandomDataNode); + scheduleNow(this::disconnectRandomDataNode); } if (randomBoolean()) { scheduleClearDisruptionNow(); @@ -246,11 +246,11 @@ public void testSnapshotWithNodeDisconnects() { masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { for (int i = 0; i < randomIntBetween(0, dataNodes); ++i) { - deterministicTaskQueue.scheduleNow(this::disconnectOrRestartDataNode); + scheduleNow(this::disconnectOrRestartDataNode); } final boolean disconnectedMaster = randomBoolean(); if (disconnectedMaster) { - deterministicTaskQueue.scheduleNow(this::disconnectOrRestartMasterNode); + scheduleNow(this::disconnectOrRestartMasterNode); } if (disconnectedMaster || randomBoolean()) { deterministicTaskQueue.scheduleAt( @@ -362,7 +362,7 @@ public void run() { if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { if (masterNodeCount > 1) { - deterministicTaskQueue.scheduleNow( + scheduleNow( () -> testClusterNodes.stopNode(masterNode)); } testClusterNodes.randomDataNodeSafe().client.admin().cluster() @@ -373,7 +373,7 @@ public void run() { new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); createdSnapshot.set(true); })); - deterministicTaskQueue.scheduleNow( + scheduleNow( () -> masterAdminClient.cluster().reroute( new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( @@ -389,8 +389,8 @@ public void run() { )); } }; - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.stopNode(currentPrimaryNode)); - deterministicTaskQueue.scheduleNow(maybeForceAllocate); + scheduleNow(() -> testClusterNodes.stopNode(currentPrimaryNode)); + scheduleNow(maybeForceAllocate); } )))))); @@ -408,8 +408,7 @@ public void run() { }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); - SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNode() - .orElseThrow(() -> new AssertionError("No data nodes running")) + final SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNodeSafe() .clusterService.state().custom(SnapshotsInProgress.TYPE); assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = masterNode.repositoriesService.repository(repoName); @@ -418,7 +417,7 @@ public void run() { } private void scheduleClearDisruptionNow() { - deterministicTaskQueue.scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); + scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); } private void disconnectOrRestartDataNode() { @@ -486,6 +485,10 @@ private void setupTestCluster(int masterNodes, int dataNodes) { startCluster(); } + private void scheduleNow(Runnable runnable) { + deterministicTaskQueue.scheduleNow(runnable); + } + private static Settings defaultIndexSettings(int shards) { // TODO: randomize replica count settings once recovery operations aren't blocking anymore return Settings.builder() @@ -730,13 +733,6 @@ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() { mockTransport = new DisruptableMockTransport(node, logger) { @Override protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { - if (node.getName().equals(destination.getName())) { - return ConnectionStatus.CONNECTED; - } - if (testClusterNodes.nodes.containsKey(node.getName()) == false - || testClusterNodes.nodes.containsKey(destination.getName()) == false) { - return ConnectionStatus.DISCONNECTED; - } return disruption.get().disrupt(node.getName(), destination.getName()) ? ConnectionStatus.DISCONNECTED : ConnectionStatus.CONNECTED; } @@ -750,7 +746,7 @@ protected Optional getDisruptableMockTransport(Transpo @Override protected void execute(Runnable runnable) { - deterministicTaskQueue.scheduleNow(CoordinatorTests.onNodeLog(getLocalNode(), runnable)); + scheduleNow(CoordinatorTests.onNodeLog(getLocalNode(), runnable)); } }; transportService = mockTransport.createTransportService( @@ -835,15 +831,14 @@ protected void assertSnapshotOrGenericThread() { final ShardStateAction shardStateAction = new ShardStateAction( clusterService, transportService, allocationService, new RoutingService(clusterService, allocationService), - deterministicTaskQueue.getThreadPool() + threadPool ); indicesClusterStateService = new IndicesClusterStateService( settings, indicesService, clusterService, threadPool, - new PeerRecoveryTargetService( - deterministicTaskQueue.getThreadPool(), transportService, recoverySettings, clusterService), + new PeerRecoveryTargetService(threadPool, transportService, recoverySettings, clusterService), shardStateAction, new NodeMappingRefreshAction(transportService, new MetaDataMappingService(clusterService, indicesService)), repositoriesService, @@ -981,12 +976,19 @@ public void connectToNodes(DiscoveryNodes discoveryNodes) { } } - private static final class DisconnectedNodes extends NetworkDisruption.DisruptedLinks { + private final class DisconnectedNodes extends NetworkDisruption.DisruptedLinks { private final Set disconnected = new HashSet<>(); @Override public boolean disrupt(String node1, String node2) { + if (node1.equals(node2)) { + return false; + } + if (testClusterNodes.nodes.containsKey(node1) == false + || testClusterNodes.nodes.containsKey(node2) == false) { + return true; + } return disconnected.contains(node1) || disconnected.contains(node2); } From 09c7554fa0a617f50ec208f359963d41ea2a67b5 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 00:07:57 +0100 Subject: [PATCH 45/62] another reproducer --- .../java/org/elasticsearch/snapshots/SnapshotsServiceTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 7cc03cc6f7d83..a1fc480cf728b 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -153,6 +153,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; +// -Dtests.seed=45B12C2784692942 public class SnapshotsServiceTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; From 3bd29044d524de02353139b687e4ee837121e57a Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 07:38:58 +0100 Subject: [PATCH 46/62] Revert "Revert "Revert "Revert "Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its"""""" This reverts commit a3bc9fc5b9e7852273e95d02b593b31b8f05042a. --- .../routing/RoutingChangesObserver.java | 49 ++++++ .../routing/allocation/AllocationService.java | 19 ++- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ++++++++++++++++++ 5 files changed, 219 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 883b4c22f7fc0..0c0d3c5a099f9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,6 +19,8 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.index.shard.ShardId; + /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -211,4 +213,51 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } + + abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { + + @Override + public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { + onChanged(unassignedShard.shardId()); + } + + @Override + public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { + onChanged(initializingShard.shardId()); + } + @Override + public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { + onChanged(startedShard.shardId()); + } + @Override + public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { + onChanged(unassignedShard.shardId()); + } + @Override + public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { + onChanged(failedShard.shardId()); + } + @Override + public void relocationCompleted(ShardRouting removedRelocationSource) { + onChanged(removedRelocationSource.shardId()); + } + @Override + public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { + onChanged(removedReplicaRelocationSource.shardId()); + } + @Override + public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { + onChanged(startedPrimaryShard.shardId()); + } + @Override + public void replicaPromoted(ShardRouting replicaShard) { + onChanged(replicaShard.shardId()); + } + @Override + public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { + onChanged(oldReplica.shardId()); + } + + protected abstract void onChanged(ShardId shardId); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 59f43a193ddc8..389eec6ed5a87 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -136,15 +137,29 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); + ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); - newStateBuilder.customs(customsBuilder.build()); } } + final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + SnapshotsInProgress updatedSnapshotsInProgress = + allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); + if (updatedSnapshotsInProgress != snapshotsInProgress) { + if (customsBuilder == null) { + customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + } + customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); + } + } + if (customsBuilder != null) { + newStateBuilder.customs(customsBuilder.build()); + } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index e0be712a230c7..f178ca18845ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -76,11 +78,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); + private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater ); - /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -251,6 +253,14 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } + /** + * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes + */ + public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, + RoutingTable newRoutingTable) { + return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); + } + /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 290589a5649a7..854fc267265b8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -325,7 +325,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to starts + // We have new shards to start if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 9b788f2ce0447..aa094b1f389be 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -45,6 +46,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -676,12 +678,28 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); + // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here + assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } + private boolean assertConsistency(ClusterState state) { + SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); + if (snapshotsInProgress != null) { + assert snapshotsInProgress == updateWithRoutingTable( + snapshotsInProgress.entries().stream().flatMap(entry -> { + Iterable iterable = () -> entry.shards().keysIt(); + return StreamSupport.stream(iterable.spliterator(), false); + }).collect(Collectors.toSet()), + snapshotsInProgress, state.routingTable() + ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; + } + return true; + } + /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1586,4 +1604,126 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } + + public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { + + private final Set shardChanges = new HashSet<>(); + + public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { + return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); + } + + @Override + protected void onChanged(ShardId shardId) { + shardChanges.add(shardId); + } + } + + private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, + RoutingTable newRoutingTable) { + if (oldSnapshot == null || shardIds.isEmpty()) { + return oldSnapshot; + } + List entries = new ArrayList<>(); + boolean snapshotsInProgressChanged = false; + for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { + ImmutableOpenMap.Builder shardsBuilder = null; + for (ShardId shardId : shardIds) { + final ImmutableOpenMap shards = entry.shards(); + final ShardSnapshotStatus currentStatus = shards.get(shardId); + if (currentStatus != null && currentStatus.state().completed() == false) { + IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); + assert routingTable != null; + final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) + .map(IndexShardRoutingTable::primaryShard) + .map( + primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) + ) + .orElse(failedStatus(null, "missing shard")); + if (newStatus != currentStatus) { + if (shardsBuilder == null) { + shardsBuilder = ImmutableOpenMap.builder(shards); + } + shardsBuilder.put(shardId, newStatus); + } + } + } + if (shardsBuilder == null) { + entries.add(entry); + } else { + snapshotsInProgressChanged = true; + ImmutableOpenMap shards = shardsBuilder.build(); + entries.add( + new SnapshotsInProgress.Entry( + entry, + completed(shards.values()) ? State.SUCCESS : entry.state(), + shards + ) + ); + } + } + if (snapshotsInProgressChanged) { + return new SnapshotsInProgress(entries); + } + return oldSnapshot; + } + + private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, + final ShardRouting primaryShardRouting) { + final State currentState = currentStatus.state(); + final ShardSnapshotStatus newStatus; + if (primaryShardRouting.active() == false) { + if (primaryShardRouting.initializing() && currentState == State.WAITING) { + newStatus = currentStatus; + } else { + newStatus = failedStatus( + primaryShardRouting.currentNodeId(), + primaryShardRouting.unassignedInfo().getReason().toString() + ); + } + } else if (primaryShardRouting.started()) { + switch (currentState) { + case WAITING: + newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); + break; + case INIT: { + String currentNodeId = currentStatus.nodeId(); + assert currentNodeId != null; + if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + } + case ABORTED: + String currentNodeId = currentStatus.nodeId(); + if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { + newStatus = currentStatus; + } else { + newStatus = failedStatus(currentNodeId); + } + break; + default: + newStatus = currentStatus; + break; + } + } else { + assert primaryShardRouting.relocating(); + if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { + newStatus = failedStatus(currentStatus.nodeId()); + } else { + newStatus = currentStatus; + } + } + return newStatus; + } + + private static ShardSnapshotStatus failedStatus(String nodeId) { + return failedStatus(nodeId, "shard failed"); + } + + private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { + return new ShardSnapshotStatus(nodeId, State.FAILED, reason); + } } From 438c40acfe7f8b5f763c5970f66b169c5bb6674e Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 07:39:32 +0100 Subject: [PATCH 47/62] Revert "Revert "Revert "Revert "Revert "Revert "Revert "Merge remote-tracking branch 'elastic/feature/snapshot-resilience' into snapshot-interruption-its""""""" This reverts commit 3bd29044d524de02353139b687e4ee837121e57a. --- .../routing/RoutingChangesObserver.java | 49 ------ .../routing/allocation/AllocationService.java | 19 +-- .../routing/allocation/RoutingAllocation.java | 14 +- .../snapshots/SnapshotShardsService.java | 2 +- .../snapshots/SnapshotsService.java | 140 ------------------ 5 files changed, 5 insertions(+), 219 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java index 0c0d3c5a099f9..883b4c22f7fc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingChangesObserver.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.index.shard.ShardId; - /** * Records changes made to {@link RoutingNodes} during an allocation round. */ @@ -213,51 +211,4 @@ public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRoutin } } } - - abstract class AbstractChangedShardObserver extends AbstractRoutingChangesObserver { - - @Override - public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { - onChanged(unassignedShard.shardId()); - } - - @Override - public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { - onChanged(initializingShard.shardId()); - } - @Override - public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) { - onChanged(startedShard.shardId()); - } - @Override - public void unassignedInfoUpdated(ShardRouting unassignedShard, UnassignedInfo newUnassignedInfo) { - onChanged(unassignedShard.shardId()); - } - @Override - public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { - onChanged(failedShard.shardId()); - } - @Override - public void relocationCompleted(ShardRouting removedRelocationSource) { - onChanged(removedRelocationSource.shardId()); - } - @Override - public void relocationSourceRemoved(ShardRouting removedReplicaRelocationSource) { - onChanged(removedReplicaRelocationSource.shardId()); - } - @Override - public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { - onChanged(startedPrimaryShard.shardId()); - } - @Override - public void replicaPromoted(ShardRouting replicaShard) { - onChanged(replicaShard.shardId()); - } - @Override - public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { - onChanged(oldReplica.shardId()); - } - - protected abstract void onChanged(ShardId shardId); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 389eec6ed5a87..59f43a193ddc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.AutoExpandReplicas; @@ -137,29 +136,15 @@ private ClusterState buildResult(ClusterState oldState, RoutingAllocation alloca final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState) .routingTable(newRoutingTable) .metaData(newMetaData); - ImmutableOpenMap.Builder customsBuilder = null; final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE); if (restoreInProgress != null) { RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress); if (updatedRestoreInProgress != restoreInProgress) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); + ImmutableOpenMap.Builder customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress); + newStateBuilder.customs(customsBuilder.build()); } } - final SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - SnapshotsInProgress updatedSnapshotsInProgress = - allocation.updateSnapshotsWithRoutingChanges(snapshotsInProgress, newRoutingTable); - if (updatedSnapshotsInProgress != snapshotsInProgress) { - if (customsBuilder == null) { - customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms()); - } - customsBuilder.put(SnapshotsInProgress.TYPE, updatedSnapshotsInProgress); - } - } - if (customsBuilder != null) { - newStateBuilder.customs(customsBuilder.build()); - } return newStateBuilder.build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index f178ca18845ae..e0be712a230c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.RestoreInProgress; -import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingChangesObserver; @@ -39,7 +38,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.elasticsearch.snapshots.SnapshotsService.SnapshotsInProgressUpdater; import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableSet; @@ -78,11 +76,11 @@ public class RoutingAllocation { private final IndexMetaDataUpdater indexMetaDataUpdater = new IndexMetaDataUpdater(); private final RoutingNodesChangedObserver nodesChangedObserver = new RoutingNodesChangedObserver(); private final RestoreInProgressUpdater restoreInProgressUpdater = new RestoreInProgressUpdater(); - private final SnapshotsInProgressUpdater snapshotsInProgressUpdater = new SnapshotsInProgressUpdater(); private final RoutingChangesObserver routingChangesObserver = new RoutingChangesObserver.DelegatingRoutingChangesObserver( - nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater, snapshotsInProgressUpdater + nodesChangedObserver, indexMetaDataUpdater, restoreInProgressUpdater ); + /** * Creates a new {@link RoutingAllocation} * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations @@ -253,14 +251,6 @@ public RestoreInProgress updateRestoreInfoWithRoutingChanges(RestoreInProgress r return restoreInProgressUpdater.applyChanges(restoreInProgress); } - /** - * Returns updated {@link SnapshotsInProgress} based on the changes that were made to the routing nodes - */ - public SnapshotsInProgress updateSnapshotsWithRoutingChanges(SnapshotsInProgress snapshotsInProgress, - RoutingTable newRoutingTable) { - return snapshotsInProgressUpdater.applyChanges(snapshotsInProgress, newRoutingTable); - } - /** * Returns true iff changes were made to the routing nodes */ diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 854fc267265b8..290589a5649a7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -325,7 +325,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { shutdownLock.unlock(); } - // We have new shards to start + // We have new shards to starts if (newSnapshots.isEmpty() == false) { Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); for (final Map.Entry> entry : newSnapshots.entrySet()) { diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index aa094b1f389be..9b788f2ce0447 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import java.util.stream.StreamSupport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -46,7 +45,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingChangesObserver; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; @@ -678,28 +676,12 @@ public void applyClusterState(ClusterChangedEvent event) { } removeFinishedSnapshotFromClusterState(event); finalizeSnapshotDeletionFromPreviousMaster(event); - // TODO org.elasticsearch.snapshots.SharedClusterSnapshotRestoreIT.testDeleteOrphanSnapshot fails right after election here - assert event.previousState().nodes().isLocalNodeElectedMaster() || assertConsistency(event.state()); } } catch (Exception e) { logger.warn("Failed to update snapshot state ", e); } } - private boolean assertConsistency(ClusterState state) { - SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null) { - assert snapshotsInProgress == updateWithRoutingTable( - snapshotsInProgress.entries().stream().flatMap(entry -> { - Iterable iterable = () -> entry.shards().keysIt(); - return StreamSupport.stream(iterable.spliterator(), false); - }).collect(Collectors.toSet()), - snapshotsInProgress, state.routingTable() - ) : "SnapshotsInProgress state [" + snapshotsInProgress + "] not in sync with routing table [" + state.routingTable() + "]."; - } - return true; - } - /** * Finalizes a snapshot deletion in progress if the current node is the master but it * was not master in the previous cluster state and there is still a lingering snapshot @@ -1604,126 +1586,4 @@ public interface SnapshotCompletionListener { void onSnapshotFailure(Snapshot snapshot, Exception e); } - - public static final class SnapshotsInProgressUpdater extends RoutingChangesObserver.AbstractChangedShardObserver { - - private final Set shardChanges = new HashSet<>(); - - public SnapshotsInProgress applyChanges(SnapshotsInProgress oldSnapshot, RoutingTable newRoutingTable) { - return updateWithRoutingTable(shardChanges, oldSnapshot, newRoutingTable); - } - - @Override - protected void onChanged(ShardId shardId) { - shardChanges.add(shardId); - } - } - - private static SnapshotsInProgress updateWithRoutingTable(Set shardIds, SnapshotsInProgress oldSnapshot, - RoutingTable newRoutingTable) { - if (oldSnapshot == null || shardIds.isEmpty()) { - return oldSnapshot; - } - List entries = new ArrayList<>(); - boolean snapshotsInProgressChanged = false; - for (SnapshotsInProgress.Entry entry : oldSnapshot.entries()) { - ImmutableOpenMap.Builder shardsBuilder = null; - for (ShardId shardId : shardIds) { - final ImmutableOpenMap shards = entry.shards(); - final ShardSnapshotStatus currentStatus = shards.get(shardId); - if (currentStatus != null && currentStatus.state().completed() == false) { - IndexShardRoutingTable routingTable = newRoutingTable.shardRoutingTableOrNull(shardId); - assert routingTable != null; - final ShardSnapshotStatus newStatus = Optional.ofNullable(routingTable) - .map(IndexShardRoutingTable::primaryShard) - .map( - primaryShardRouting -> determineShardSnapshotStatus(currentStatus, primaryShardRouting) - ) - .orElse(failedStatus(null, "missing shard")); - if (newStatus != currentStatus) { - if (shardsBuilder == null) { - shardsBuilder = ImmutableOpenMap.builder(shards); - } - shardsBuilder.put(shardId, newStatus); - } - } - } - if (shardsBuilder == null) { - entries.add(entry); - } else { - snapshotsInProgressChanged = true; - ImmutableOpenMap shards = shardsBuilder.build(); - entries.add( - new SnapshotsInProgress.Entry( - entry, - completed(shards.values()) ? State.SUCCESS : entry.state(), - shards - ) - ); - } - } - if (snapshotsInProgressChanged) { - return new SnapshotsInProgress(entries); - } - return oldSnapshot; - } - - private static ShardSnapshotStatus determineShardSnapshotStatus(final ShardSnapshotStatus currentStatus, - final ShardRouting primaryShardRouting) { - final State currentState = currentStatus.state(); - final ShardSnapshotStatus newStatus; - if (primaryShardRouting.active() == false) { - if (primaryShardRouting.initializing() && currentState == State.WAITING) { - newStatus = currentStatus; - } else { - newStatus = failedStatus( - primaryShardRouting.currentNodeId(), - primaryShardRouting.unassignedInfo().getReason().toString() - ); - } - } else if (primaryShardRouting.started()) { - switch (currentState) { - case WAITING: - newStatus = new ShardSnapshotStatus(primaryShardRouting.currentNodeId()); - break; - case INIT: { - String currentNodeId = currentStatus.nodeId(); - assert currentNodeId != null; - if (primaryShardRouting.currentNodeId().equals(currentNodeId)) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - } - case ABORTED: - String currentNodeId = currentStatus.nodeId(); - if (currentNodeId.equals(primaryShardRouting.currentNodeId())) { - newStatus = currentStatus; - } else { - newStatus = failedStatus(currentNodeId); - } - break; - default: - newStatus = currentStatus; - break; - } - } else { - assert primaryShardRouting.relocating(); - if (currentState == State.INIT || currentStatus.state() == State.ABORTED) { - newStatus = failedStatus(currentStatus.nodeId()); - } else { - newStatus = currentStatus; - } - } - return newStatus; - } - - private static ShardSnapshotStatus failedStatus(String nodeId) { - return failedStatus(nodeId, "shard failed"); - } - - private static ShardSnapshotStatus failedStatus(String nodeId, String reason) { - return new ShardSnapshotStatus(nodeId, State.FAILED, reason); - } } From 72a96190497199999107c441c713812cc4bf5b6c Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 07:55:57 +0100 Subject: [PATCH 48/62] nicer --- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index a1fc480cf728b..7dca302e35633 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -153,7 +153,6 @@ import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; -// -Dtests.seed=45B12C2784692942 public class SnapshotsServiceTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; @@ -266,7 +265,8 @@ public void testSnapshotWithNodeDisconnects() { })))); runUntil(() -> { - final SnapshotsInProgress snapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + final SnapshotsInProgress snapshotsInProgress = testClusterNodes.randomMasterNode() + .clusterService.state().custom(SnapshotsInProgress.TYPE); return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); }, TimeUnit.MINUTES.toMillis(20L)); From 776c56d0991ea12828d0a92821bd9296678cc43c Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 08:09:58 +0100 Subject: [PATCH 49/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 7dca302e35633..777fe04c443bd 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -125,7 +125,6 @@ import org.junit.Before; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; @@ -241,7 +240,7 @@ public void testSnapshotWithNodeDisconnects() { scheduleNow(this::disconnectRandomDataNode); } if (randomBoolean()) { - scheduleClearDisruptionNow(); + scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); } masterAdminClient.cluster().prepareCreateSnapshot(repoName, snapshotName) .execute(assertNoFailureListener(() -> { @@ -253,12 +252,9 @@ public void testSnapshotWithNodeDisconnects() { scheduleNow(this::disconnectOrRestartMasterNode); } if (disconnectedMaster || randomBoolean()) { - deterministicTaskQueue.scheduleAt( - deterministicTaskQueue.getCurrentTimeMillis() - + randomLongBetween(0L, TimeUnit.MINUTES.toMillis(10L)), - () -> testClusterNodes.clearNetworkDisruptions()); + scheduleSoon(() -> testClusterNodes.clearNetworkDisruptions()); } else if (randomBoolean()) { - scheduleClearDisruptionNow(); + scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); } createdSnapshot.set(true); })); @@ -271,9 +267,10 @@ public void testSnapshotWithNodeDisconnects() { }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); - SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + final TestClusterNode randomMaster = testClusterNodes.randomMasterNode(); + SnapshotsInProgress finalSnapshotsInProgress = randomMaster.clusterService.state().custom(SnapshotsInProgress.TYPE); assertThat(finalSnapshotsInProgress.entries(), empty()); - final Repository repository = masterNode.repositoriesService.repository(repoName); + final Repository repository = randomMaster.repositoriesService.repository(repoName); Collection snapshotIds = repository.getRepositoryData().getSnapshotIds(); assertThat(snapshotIds, hasSize(1)); } @@ -363,8 +360,7 @@ public void run() { if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) { if (masterNodeCount > 1) { - scheduleNow( - () -> testClusterNodes.stopNode(masterNode)); + scheduleNow(() -> testClusterNodes.stopNode(masterNode)); } testClusterNodes.randomDataNodeSafe().client.admin().cluster() .prepareCreateSnapshot(repoName, snapshotName) @@ -381,10 +377,7 @@ public void run() { index, shardRouting.shardId().id(), otherNode.node.getName(), true) ), noopListener())); } else { - deterministicTaskQueue.scheduleAt( - deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), - this - ); + scheduleSoon(this); } } )); @@ -417,10 +410,6 @@ public void run() { assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0))); } - private void scheduleClearDisruptionNow() { - scheduleNow(() -> testClusterNodes.clearNetworkDisruptions()); - } - private void disconnectOrRestartDataNode() { if (randomBoolean()) { disconnectRandomDataNode(); @@ -486,6 +475,10 @@ private void setupTestCluster(int masterNodes, int dataNodes) { startCluster(); } + private void scheduleSoon(Runnable runnable) { + deterministicTaskQueue.scheduleAt(deterministicTaskQueue.getCurrentTimeMillis() + randomLongBetween(0, 100L), runnable); + } + private void scheduleNow(Runnable runnable) { deterministicTaskQueue.scheduleNow(runnable); } @@ -567,7 +560,7 @@ private final class TestClusterNodes { try { return newMasterNode(nodeName); } catch (IOException e) { - throw new UncheckedIOException(e); + throw new AssertionError(e); } }); } @@ -576,7 +569,7 @@ private final class TestClusterNodes { try { return newDataNode(nodeName); } catch (IOException e) { - throw new UncheckedIOException(e); + throw new AssertionError(e); } }); } @@ -758,8 +751,7 @@ public TransportRequestHandler interceptHandler( boolean forceExecution, TransportRequestHandler actualHandler) { // TODO: Remove this hack once recoveries are async and can be used in these tests if (action.startsWith("internal:index/shard/recovery")) { - return (request, channel, task) -> deterministicTaskQueue.scheduleAt( - deterministicTaskQueue.getCurrentTimeMillis() + 20L, + return (request, channel, task) -> scheduleSoon( new AbstractRunnable() { @Override protected void doRun() throws Exception { @@ -911,16 +903,15 @@ public void restart() { final ClusterState oldState = this.clusterService.state(); stop(); testClusterNodes.nodes.remove(node.getName()); - deterministicTaskQueue.scheduleAt(randomFrom(0L, deterministicTaskQueue.getCurrentTimeMillis()), () -> { + scheduleSoon(() -> { try { - final TestClusterNode restartedNode = - new TestClusterNode( - new DiscoveryNode(node.getName(), node.getId(), node.getAddress(), emptyMap(), - node.getRoles(), Version.CURRENT), disruption); + final TestClusterNode restartedNode = new TestClusterNode( + new DiscoveryNode(node.getName(), node.getId(), node.getAddress(), emptyMap(), + node.getRoles(), Version.CURRENT), disruption); testClusterNodes.nodes.put(node.getName(), restartedNode); restartedNode.start(oldState); } catch (IOException e) { - throw new UncheckedIOException(e); + throw new AssertionError(e); } }); } From 013aea1763f033eeb4657c1a34a56973779cecd9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 08:31:24 +0100 Subject: [PATCH 50/62] fix hitting dead node --- .../java/org/elasticsearch/snapshots/SnapshotsServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 777fe04c443bd..e570745eeba98 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -371,7 +371,7 @@ public void run() { createdSnapshot.set(true); })); scheduleNow( - () -> masterAdminClient.cluster().reroute( + () -> testClusterNodes.randomMasterNode().client.admin().cluster().reroute( new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true) From 0a3ed436b948680f18e99e90d320835e0f6de7ee Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 09:02:33 +0100 Subject: [PATCH 51/62] fix hitting dead node --- .../snapshots/SnapshotsServiceTests.java | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index e570745eeba98..74519254605f8 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -261,13 +261,24 @@ public void testSnapshotWithNodeDisconnects() { })))); runUntil(() -> { - final SnapshotsInProgress snapshotsInProgress = testClusterNodes.randomMasterNode() - .clusterService.state().custom(SnapshotsInProgress.TYPE); - return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); + final Optional randomMaster = testClusterNodes.randomMasterNode(); + if (randomMaster.isPresent()) { + final SnapshotsInProgress snapshotsInProgress = randomMaster.get().clusterService.state().custom(SnapshotsInProgress.TYPE); + return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); + } + return false; + }, TimeUnit.MINUTES.toMillis(20L)); + + testClusterNodes.clearNetworkDisruptions(); + runUntil(() -> { + final List versions = testClusterNodes.nodes.values().stream() + .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); + return versions.size() == 1L; }, TimeUnit.MINUTES.toMillis(20L)); assertTrue(createdSnapshot.get()); - final TestClusterNode randomMaster = testClusterNodes.randomMasterNode(); + final TestClusterNode randomMaster = testClusterNodes.randomMasterNode() + .orElseThrow(() -> new AssertionError("expected to find at least one active master node")); SnapshotsInProgress finalSnapshotsInProgress = randomMaster.clusterService.state().custom(SnapshotsInProgress.TYPE); assertThat(finalSnapshotsInProgress.entries(), empty()); final Repository repository = randomMaster.repositoriesService.repository(repoName); @@ -371,7 +382,7 @@ public void run() { createdSnapshot.set(true); })); scheduleNow( - () -> testClusterNodes.randomMasterNode().client.admin().cluster().reroute( + () -> testClusterNodes.randomMasterNodeSafe().client.admin().cluster().reroute( new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true) @@ -389,9 +400,13 @@ public void run() { )))))); runUntil(() -> { - final SnapshotsInProgress snapshotsInProgress = - testClusterNodes.randomMasterNode().clusterService.state().custom(SnapshotsInProgress.TYPE); - return (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) && createdSnapshot.get(); + final Optional randomMaster = testClusterNodes.randomMasterNode(); + if (randomMaster.isPresent()) { + final SnapshotsInProgress snapshotsInProgress = + randomMaster.get().clusterService.state().custom(SnapshotsInProgress.TYPE); + return (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) && createdSnapshot.get(); + } + return false; }, TimeUnit.MINUTES.toMillis(20L)); testClusterNodes.clearNetworkDisruptions(); @@ -419,11 +434,13 @@ private void disconnectOrRestartDataNode() { } private void disconnectOrRestartMasterNode() { - if (randomBoolean()) { - testClusterNodes.disconnectNode(testClusterNodes.randomMasterNode()); - } else { - testClusterNodes.randomMasterNode().restart(); - } + testClusterNodes.randomMasterNode().ifPresent(masterNode -> { + if (randomBoolean()) { + testClusterNodes.disconnectNode(masterNode); + } else { + masterNode.restart(); + } + }); } private void disconnectRandomDataNode() { @@ -594,12 +611,15 @@ private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws Collections.singleton(role), Version.CURRENT), this::getDisruption); } - public TestClusterNode randomMasterNode() { + public TestClusterNode randomMasterNodeSafe() { + return randomMasterNode().orElseThrow(() -> new AssertionError("Expected to find at least one connected master node")); + } + + public Optional randomMasterNode() { // Select from sorted list of data-nodes here to not have deterministic behaviour - return randomFrom( - testClusterNodes.nodes.values().stream().filter(n -> n.node.isMasterNode()) - .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()) - ); + final List masterNodes = testClusterNodes.nodes.values().stream().filter(n -> n.node.isMasterNode()) + .sorted(Comparator.comparing(n -> n.node.getName())).collect(Collectors.toList()); + return masterNodes.isEmpty() ? Optional.empty() : Optional.of(randomFrom(masterNodes)); } public void stopNode(TestClusterNode node) { From ff7cfdf6e396bef72a5b5a8df82ca863b6cd76e7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 09:05:32 +0100 Subject: [PATCH 52/62] nicer --- .../snapshots/SnapshotsServiceTests.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 74519254605f8..656fd229ae3e1 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -269,12 +269,7 @@ public void testSnapshotWithNodeDisconnects() { return false; }, TimeUnit.MINUTES.toMillis(20L)); - testClusterNodes.clearNetworkDisruptions(); - runUntil(() -> { - final List versions = testClusterNodes.nodes.values().stream() - .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); - return versions.size() == 1L; - }, TimeUnit.MINUTES.toMillis(20L)); + clearDisruptionsAndAwaitSync(); assertTrue(createdSnapshot.get()); final TestClusterNode randomMaster = testClusterNodes.randomMasterNode() @@ -409,12 +404,7 @@ public void run() { return false; }, TimeUnit.MINUTES.toMillis(20L)); - testClusterNodes.clearNetworkDisruptions(); - runUntil(() -> { - final List versions = testClusterNodes.nodes.values().stream() - .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); - return versions.size() == 1L; - }, TimeUnit.MINUTES.toMillis(20L)); + clearDisruptionsAndAwaitSync(); assertTrue(createdSnapshot.get()); final SnapshotsInProgress finalSnapshotsInProgress = testClusterNodes.randomDataNodeSafe() @@ -425,6 +415,15 @@ public void run() { assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0))); } + private void clearDisruptionsAndAwaitSync() { + testClusterNodes.clearNetworkDisruptions(); + runUntil(() -> { + final List versions = testClusterNodes.nodes.values().stream() + .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); + return versions.size() == 1L; + }, TimeUnit.MINUTES.toMillis(20L)); + } + private void disconnectOrRestartDataNode() { if (randomBoolean()) { disconnectRandomDataNode(); From 634a71521c75e0b43764e612a3b6396b30884888 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 09:17:18 +0100 Subject: [PATCH 53/62] nicer --- .../elasticsearch/snapshots/SnapshotsServiceTests.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 656fd229ae3e1..c7d3622ed9818 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -326,6 +326,10 @@ public void testConcurrentSnapshotCreateAndDelete() { assertEquals(0, snapshotInfo.failedShards()); } + /** + * Simulates concurrent restarts of data and master nodes as well as relocating a primary shard, while starting and subsequently + * deleting a snapshot. + */ public void testSnapshotPrimaryRelocations() { final int masterNodeCount = randomFrom(1, 3, 5); setupTestCluster(masterNodeCount, randomIntBetween(2, 10)); @@ -989,6 +993,9 @@ public void connectToNodes(DiscoveryNodes discoveryNodes) { private final class DisconnectedNodes extends NetworkDisruption.DisruptedLinks { + /** + * Node names that are disconnected from all other nodes. + */ private final Set disconnected = new HashSet<>(); @Override @@ -996,6 +1003,7 @@ public boolean disrupt(String node1, String node2) { if (node1.equals(node2)) { return false; } + // Check if both nodes are still part of the cluster if (testClusterNodes.nodes.containsKey(node1) == false || testClusterNodes.nodes.containsKey(node2) == false) { return true; From 03e198470a4e7641852a125e42d240edac97b4dd Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 09:21:51 +0100 Subject: [PATCH 54/62] nicer --- .../snapshots/SnapshotShardsService.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 290589a5649a7..5cf4f857ce3a1 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -296,15 +296,11 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } else { final ImmutableOpenMap clusterStateShards = entry.shards(); - if (clusterStateShards.isEmpty()) { - - } else { - for (ObjectObjectCursor curr : clusterStateShards) { - if (curr.value.state() == State.ABORTED) { - // due to CS batching we might have missed the INIT state and straight went into ABORTED - // notify master that abort has completed by moving to FAILED - notifyFailedSnapshotShard(entry.snapshot(), curr.key, localNodeId, curr.value.reason()); - } + for (ObjectObjectCursor curr : clusterStateShards) { + if (curr.value.state() == State.ABORTED) { + // due to CS batching we might have missed the INIT state and straight went into ABORTED + // notify master that abort has completed by moving to FAILED + notifyFailedSnapshotShard(entry.snapshot(), curr.key, localNodeId, curr.value.reason()); } } } From bc450e851d0ebc70c277a4a167e0b5c1449f57cf Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 12:01:10 +0100 Subject: [PATCH 55/62] CR: no shards when init --- .../org/elasticsearch/snapshots/SnapshotsService.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 9b788f2ce0447..9d119f392655d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1197,11 +1197,10 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (state == State.INIT) { // snapshot is still initializing, mark it as aborted shards = snapshotEntry.shards(); - if (shards.isEmpty()) { - // No shards in this snapshot, we delete it right away since the SnapshotShardsService - // has no work to do. - endSnapshot(snapshotEntry); - } + assert shards.isEmpty(); + // No shards in this snapshot, we delete it right away since the SnapshotShardsService + // has no work to do. + endSnapshot(snapshotEntry); } else if (state == State.STARTED) { // snapshot is started - mark every non completed shard as aborted final ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(); From 803003f92f3d2a448719c2dbd04cd51256e9bb44 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 12:13:29 +0100 Subject: [PATCH 56/62] nicer writable register getter --- .../snapshots/SnapshotsServiceTests.java | 11 ++++++++++- .../test/disruption/DisruptableMockTransport.java | 11 ----------- .../elasticsearch/test/transport/MockTransport.java | 9 +++++++-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index c7d3622ed9818..c30618c9b2033 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ESAllocationTestCase; @@ -79,6 +80,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; @@ -141,6 +143,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -698,7 +701,8 @@ private final class TestClusterNode { private final Logger logger = LogManager.getLogger(TestClusterNode.class); - private final NamedWriteableRegistry namedWriteableRegistry = DisruptableMockTransport.writeableRegistry(); + private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Stream.concat( + ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()).collect(Collectors.toList())); private final TransportService transportService; @@ -765,6 +769,11 @@ protected Optional getDisruptableMockTransport(Transpo protected void execute(Runnable runnable) { scheduleNow(CoordinatorTests.onNodeLog(getLocalNode(), runnable)); } + + @Override + protected NamedWriteableRegistry writeableRegistry() { + return namedWriteableRegistry; + } }; transportService = mockTransport.createTransportService( settings, deterministicTaskQueue.getThreadPool(runnable -> CoordinatorTests.onNodeLog(node, runnable)), diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java index 612ff8ab09bf2..d750a8256b8bc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/DisruptableMockTransport.java @@ -21,12 +21,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -49,8 +46,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.elasticsearch.test.ESTestCase.copyWriteable; import static org.elasticsearch.transport.TransportService.HANDSHAKE_ACTION_NAME; @@ -256,12 +251,6 @@ public String toString() { } } - public static NamedWriteableRegistry writeableRegistry() { - return new NamedWriteableRegistry( - Stream.concat(ClusterModule.getNamedWriteables().stream(), NetworkModule.getNamedWriteables().stream()) - .collect(Collectors.toList())); - } - public enum ConnectionStatus { CONNECTED, DISCONNECTED, // network requests to or from this node throw a ConnectTransportException diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java index ae61ca1e192a3..a6dbd1561936e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransport.java @@ -20,6 +20,7 @@ package org.elasticsearch.test.transport; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Randomness; @@ -30,12 +31,12 @@ import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.disruption.DisruptableMockTransport; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.CloseableConnection; import org.elasticsearch.transport.ConnectionManager; @@ -99,7 +100,7 @@ public void handleResponse(final long reque try (BytesStreamOutput output = new BytesStreamOutput()) { response.writeTo(output); deliveredResponse = transportResponseHandler.read( - new NamedWriteableAwareStreamInput(output.bytes().streamInput(), DisruptableMockTransport.writeableRegistry())); + new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writeableRegistry())); } catch (IOException | UnsupportedOperationException e) { throw new AssertionError("failed to serialize/deserialize response " + response, e); } @@ -278,4 +279,8 @@ public boolean removeMessageListener(TransportMessageListener listener) { } return false; } + + protected NamedWriteableRegistry writeableRegistry() { + return new NamedWriteableRegistry(ClusterModule.getNamedWriteables()); + } } From 808c99a1c13caa64578ff351474e150815b81c01 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 22 Jan 2019 14:09:15 +0100 Subject: [PATCH 57/62] CR: cache failed notifications --- .../snapshots/SnapshotShardsService.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 5cf4f857ce3a1..0408ac359e922 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -71,9 +71,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; @@ -112,6 +115,9 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements private volatile Map> shardSnapshots = emptyMap(); + // A map of snapshots to the shardIds that we already reported to the master as failed + private volatile Map> remoteFailedShardsCache = emptyMap(); + private final SnapshotStateExecutor snapshotStateExecutor = new SnapshotStateExecutor(); private final UpdateSnapshotStatusAction updateSnapshotStatusHandler; @@ -237,6 +243,7 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { // Now go through all snapshots and update existing or create missing final String localNodeId = event.state().nodes().getLocalNodeId(); final Map> snapshotIndices = new HashMap<>(); + final Map> newFailedShardsCache = new HashMap<>(); if (snapshotsInProgress != null) { for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { snapshotIndices.put(entry.snapshot(), @@ -296,11 +303,18 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } else { final ImmutableOpenMap clusterStateShards = entry.shards(); + final Snapshot snapshot = entry.snapshot(); + final Set cachedFailedShards = remoteFailedShardsCache.getOrDefault(snapshot, Collections.emptySet()); for (ObjectObjectCursor curr : clusterStateShards) { + // due to CS batching we might have missed the INIT state and straight went into ABORTED + // notify master that abort has completed by moving to FAILED if (curr.value.state() == State.ABORTED) { - // due to CS batching we might have missed the INIT state and straight went into ABORTED - // notify master that abort has completed by moving to FAILED - notifyFailedSnapshotShard(entry.snapshot(), curr.key, localNodeId, curr.value.reason()); + final ShardId shardId = curr.key; + final Set newFailedShardIds = + newFailedShardsCache.computeIfAbsent(snapshot, k -> new HashSet<>(cachedFailedShards)); + if (newFailedShardIds.add(shardId)) { + notifyFailedSnapshotShard(snapshot, shardId, localNodeId, curr.value.reason()); + } } } } @@ -308,6 +322,8 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } + remoteFailedShardsCache = unmodifiableMap(newFailedShardsCache); + // Update the list of snapshots that we saw and tried to started // If startup of these shards fails later, we don't want to try starting these shards again shutdownLock.lock(); From 1366efc8d83d17394f426e3969bfab3de4c251d1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 23 Jan 2019 12:51:34 +0100 Subject: [PATCH 58/62] CR: return false from terminate --- .../cluster/coordination/MockSinglePrioritizingExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 032c719b4997a..7109edd58b6c6 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -56,7 +56,7 @@ protected void afterExecute(Runnable r, Throwable t) { // Override awaitTermination to not get blocked on termination condition @Override public boolean awaitTermination(long timeout, TimeUnit unit) { - return true; + return false; } private static final class KillWorkerError extends Error { From 326cd5648d7b47d09abc2a360740d4d48c52d3d4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 23 Jan 2019 12:57:14 +0100 Subject: [PATCH 59/62] CR: lower timeout --- .../org/elasticsearch/snapshots/SnapshotsServiceTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index fe4cceb03bb45..904ee03d46464 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -270,7 +270,7 @@ public void testSnapshotWithNodeDisconnects() { return snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty(); } return false; - }, TimeUnit.MINUTES.toMillis(20L)); + }, TimeUnit.MINUTES.toMillis(1L)); clearDisruptionsAndAwaitSync(); @@ -409,7 +409,7 @@ public void run() { return (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) && createdSnapshot.get(); } return false; - }, TimeUnit.MINUTES.toMillis(20L)); + }, TimeUnit.MINUTES.toMillis(1L)); clearDisruptionsAndAwaitSync(); @@ -428,7 +428,7 @@ private void clearDisruptionsAndAwaitSync() { final List versions = testClusterNodes.nodes.values().stream() .map(n -> n.clusterService.state().version()).distinct().collect(Collectors.toList()); return versions.size() == 1L; - }, TimeUnit.MINUTES.toMillis(20L)); + }, TimeUnit.MINUTES.toMillis(1L)); } private void disconnectOrRestartDataNode() { From af35d2f2b2f066effe80dc6919267f18988e5e29 Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 30 Jan 2019 20:43:48 +0100 Subject: [PATCH 60/62] nicer noop ActionListener --- .../elasticsearch/action/ActionListener.java | 25 +++++++++ .../snapshots/SnapshotShardsService.java | 53 +++++++++++-------- .../snapshots/SnapshotsServiceTests.java | 17 ++---- 3 files changed, 58 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index 34d90035ad6cc..6b270b3f8269d 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -32,6 +32,21 @@ * A listener for action responses or failures. */ public interface ActionListener { + + /** + * ActionListener that does nothing. + * @see #noop() + */ + ActionListener NOOP = new ActionListener() { + @Override + public void onResponse(Object response) { + } + + @Override + public void onFailure(Exception e) { + } + }; + /** * Handle action response. This response may constitute a failure or a * success but it is up to the listener to make that decision. @@ -43,6 +58,16 @@ public interface ActionListener { */ void onFailure(Exception e); + /** + * Returns an ActionListener that does nothing. + * @param the type of the response + * @return ActionListener that does nothing + */ + @SuppressWarnings("unchecked") + static ActionListener noop() { + return (ActionListener) NOOP; + } + /** * Creates a listener that listens for a response (or failure) and executes the * corresponding consumer when the response (or failure) is received. diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 0408ac359e922..44b6148efb382 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -67,16 +67,17 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.EmptyTransportResponseHandler; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequestDeduplicator; +import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; @@ -88,7 +89,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; -import static org.elasticsearch.transport.EmptyTransportResponseHandler.INSTANCE_SAME; /** * This service runs on data and master nodes and controls currently snapshotted shards on these nodes. It is responsible for @@ -116,7 +116,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements private volatile Map> shardSnapshots = emptyMap(); // A map of snapshots to the shardIds that we already reported to the master as failed - private volatile Map> remoteFailedShardsCache = emptyMap(); + private final TransportRequestDeduplicator remoteFailedRequestDeduplicator = + new TransportRequestDeduplicator<>(); private final SnapshotStateExecutor snapshotStateExecutor = new SnapshotStateExecutor(); private final UpdateSnapshotStatusAction updateSnapshotStatusHandler; @@ -243,7 +244,6 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { // Now go through all snapshots and update existing or create missing final String localNodeId = event.state().nodes().getLocalNodeId(); final Map> snapshotIndices = new HashMap<>(); - final Map> newFailedShardsCache = new HashMap<>(); if (snapshotsInProgress != null) { for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { snapshotIndices.put(entry.snapshot(), @@ -302,19 +302,12 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } } else { - final ImmutableOpenMap clusterStateShards = entry.shards(); final Snapshot snapshot = entry.snapshot(); - final Set cachedFailedShards = remoteFailedShardsCache.getOrDefault(snapshot, Collections.emptySet()); - for (ObjectObjectCursor curr : clusterStateShards) { + for (ObjectObjectCursor curr : entry.shards()) { // due to CS batching we might have missed the INIT state and straight went into ABORTED // notify master that abort has completed by moving to FAILED if (curr.value.state() == State.ABORTED) { - final ShardId shardId = curr.key; - final Set newFailedShardIds = - newFailedShardsCache.computeIfAbsent(snapshot, k -> new HashSet<>(cachedFailedShards)); - if (newFailedShardIds.add(shardId)) { - notifyFailedSnapshotShard(snapshot, shardId, localNodeId, curr.value.reason()); - } + notifyFailedSnapshotShard(snapshot, curr.key, localNodeId, curr.value.reason()); } } } @@ -322,8 +315,6 @@ private void processIndexShardSnapshots(ClusterChangedEvent event) { } } - remoteFailedShardsCache = unmodifiableMap(newFailedShardsCache); - // Update the list of snapshots that we saw and tried to started // If startup of these shards fails later, we don't want to try starting these shards again shutdownLock.lock(); @@ -539,12 +530,28 @@ void notifyFailedSnapshotShard(final Snapshot snapshot, final ShardId shardId, f /** Updates the shard snapshot status by sending a {@link UpdateIndexShardSnapshotStatusRequest} to the master node */ void sendSnapshotShardUpdate(final Snapshot snapshot, final ShardId shardId, final ShardSnapshotStatus status) { - try { - UpdateIndexShardSnapshotStatusRequest request = new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status); - transportService.sendRequest(transportService.getLocalNode(), UPDATE_SNAPSHOT_STATUS_ACTION_NAME, request, INSTANCE_SAME); - } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", snapshot, status), e); - } + remoteFailedRequestDeduplicator.executeOnce( + new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status), + ActionListener.noop(), + (req, reqListener) -> { + try { + transportService.sendRequest(transportService.getLocalNode(), UPDATE_SNAPSHOT_STATUS_ACTION_NAME, req, + new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + @Override + public void handleResponse(TransportResponse.Empty response) { + reqListener.onResponse(null); + } + + @Override + public void handleException(TransportException exp) { + reqListener.onFailure(exp); + } + }); + } catch (Exception e) { + logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", snapshot, status), e); + } + } + ); } /** diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java index 957dc77197bd4..1e3a86c523373 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java @@ -381,7 +381,8 @@ public void run() { .execute(ActionListener.wrap(() -> { testClusterNodes.randomDataNodeSafe().client.admin().cluster() .deleteSnapshot( - new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); + new DeleteSnapshotRequest(repoName, snapshotName), + ActionListener.noop()); createdSnapshot.set(true); })); scheduleNow( @@ -389,7 +390,7 @@ public void run() { new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true) - ), noopListener())); + ), ActionListener.noop())); } else { scheduleSoon(this); } @@ -538,18 +539,6 @@ public void onFailure(final Exception e) { }; } - private static ActionListener noopListener() { - return new ActionListener() { - @Override - public void onResponse(final T t) { - } - - @Override - public void onFailure(final Exception e) { - } - }; - } - /** * Create a {@link Environment} with random path.home and path.repo **/ From 8b25eac7c62d91c5159ff31ccc7c6d0f4afc3e70 Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 30 Jan 2019 20:52:55 +0100 Subject: [PATCH 61/62] remove noisy change --- .../cluster/coordination/MockSinglePrioritizingExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 7109edd58b6c6..bcc10f1521b29 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -53,9 +53,9 @@ protected void afterExecute(Runnable r, Throwable t) { throw new KillWorkerError(); } - // Override awaitTermination to not get blocked on termination condition @Override public boolean awaitTermination(long timeout, TimeUnit unit) { + // ensures we don't block return false; } From 69f8f7ded287122949f07feb8bd996e124e2495b Mon Sep 17 00:00:00 2001 From: Armin Date: Thu, 31 Jan 2019 08:47:47 +0100 Subject: [PATCH 62/62] CR: renaming + remove noop listener --- .../elasticsearch/action/ActionListener.java | 25 ------------ .../snapshots/SnapshotShardsService.java | 39 +++++++++++-------- ...ests.java => SnapshotResiliencyTests.java} | 19 +++++++-- 3 files changed, 37 insertions(+), 46 deletions(-) rename server/src/test/java/org/elasticsearch/snapshots/{SnapshotsServiceTests.java => SnapshotResiliencyTests.java} (99%) diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index 6b270b3f8269d..34d90035ad6cc 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -32,21 +32,6 @@ * A listener for action responses or failures. */ public interface ActionListener { - - /** - * ActionListener that does nothing. - * @see #noop() - */ - ActionListener NOOP = new ActionListener() { - @Override - public void onResponse(Object response) { - } - - @Override - public void onFailure(Exception e) { - } - }; - /** * Handle action response. This response may constitute a failure or a * success but it is up to the listener to make that decision. @@ -58,16 +43,6 @@ public void onFailure(Exception e) { */ void onFailure(Exception e); - /** - * Returns an ActionListener that does nothing. - * @param the type of the response - * @return ActionListener that does nothing - */ - @SuppressWarnings("unchecked") - static ActionListener noop() { - return (ActionListener) NOOP; - } - /** * Creates a listener that listens for a response (or failure) and executes the * corresponding consumer when the response (or failure) is received. diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 44b6148efb382..132b269b196e0 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -532,25 +532,30 @@ void notifyFailedSnapshotShard(final Snapshot snapshot, final ShardId shardId, f void sendSnapshotShardUpdate(final Snapshot snapshot, final ShardId shardId, final ShardSnapshotStatus status) { remoteFailedRequestDeduplicator.executeOnce( new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status), - ActionListener.noop(), - (req, reqListener) -> { - try { - transportService.sendRequest(transportService.getLocalNode(), UPDATE_SNAPSHOT_STATUS_ACTION_NAME, req, - new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { - @Override - public void handleResponse(TransportResponse.Empty response) { - reqListener.onResponse(null); - } + new ActionListener() { + @Override + public void onResponse(Void aVoid) { + logger.trace("[{}] [{}] updated snapshot state", snapshot, status); + } - @Override - public void handleException(TransportException exp) { - reqListener.onFailure(exp); - } - }); - } catch (Exception e) { - logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", snapshot, status), e); + @Override + public void onFailure(Exception e) { + logger.warn( + () -> new ParameterizedMessage("[{}] [{}] failed to update snapshot state", snapshot, status), e); } - } + }, + (req, reqListener) -> transportService.sendRequest(transportService.getLocalNode(), UPDATE_SNAPSHOT_STATUS_ACTION_NAME, req, + new EmptyTransportResponseHandler(ThreadPool.Names.SAME) { + @Override + public void handleResponse(TransportResponse.Empty response) { + reqListener.onResponse(null); + } + + @Override + public void handleException(TransportException exp) { + reqListener.onFailure(exp); + } + }) ); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java rename to server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 1e3a86c523373..a54155db92da1 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -156,7 +156,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.mockito.Mockito.mock; -public class SnapshotsServiceTests extends ESTestCase { +public class SnapshotResiliencyTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; @@ -381,8 +381,7 @@ public void run() { .execute(ActionListener.wrap(() -> { testClusterNodes.randomDataNodeSafe().client.admin().cluster() .deleteSnapshot( - new DeleteSnapshotRequest(repoName, snapshotName), - ActionListener.noop()); + new DeleteSnapshotRequest(repoName, snapshotName), noopListener()); createdSnapshot.set(true); })); scheduleNow( @@ -390,7 +389,7 @@ public void run() { new ClusterRerouteRequest().add( new AllocateEmptyPrimaryAllocationCommand( index, shardRouting.shardId().id(), otherNode.node.getName(), true) - ), ActionListener.noop())); + ), noopListener())); } else { scheduleSoon(this); } @@ -539,6 +538,18 @@ public void onFailure(final Exception e) { }; } + private static ActionListener noopListener() { + return new ActionListener() { + @Override + public void onResponse(final T t) { + } + + @Override + public void onFailure(final Exception e) { + } + }; + } + /** * Create a {@link Environment} with random path.home and path.repo **/