Merge branch 'master' into feature/client_aggs_parsing

Author: cbuescher

buildSrc/version.properties

@@ -1,5 +1,5 @@
 # When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-elasticsearch     = 6.0.0-alpha1
+elasticsearch     = 6.0.0-alpha2
 lucene            = 7.0.0-snapshot-89f6d17
 
 # optional dependencies

core/src/main/java/org/elasticsearch/Version.java

@@ -87,7 +87,10 @@ public class Version implements Comparable<Version> {
     public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
     public static final Version V_6_0_0_alpha1_UNRELEASED =
             new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0);
-    public static final Version CURRENT = V_6_0_0_alpha1_UNRELEASED;
+    public static final int V_6_0_0_alpha2_ID_UNRELEASED = 6000002;
+    public static final Version V_6_0_0_alpha2_UNRELEASED =
+            new Version(V_6_0_0_alpha2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_7_0_0);
+    public static final Version CURRENT = V_6_0_0_alpha2_UNRELEASED;
 
     // unreleased versions must be added to the above list with the suffix _UNRELEASED (with the exception of CURRENT)
 
@@ -102,6 +105,8 @@ public static Version readVersion(StreamInput in) throws IOException {
 
     public static Version fromId(int id) {
         switch (id) {
+            case V_6_0_0_alpha2_ID_UNRELEASED:
+                return V_6_0_0_alpha2_UNRELEASED;
             case V_6_0_0_alpha1_ID_UNRELEASED:
                 return V_6_0_0_alpha1_UNRELEASED;
             case V_5_5_0_ID_UNRELEASED:

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequest.java

@@ -28,6 +28,7 @@
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.snapshots.SnapshotInfo.VERBOSE_INTRODUCED;
 
 /**
  * Get snapshot request
@@ -43,6 +44,8 @@ public class GetSnapshotsRequest extends MasterNodeRequest<GetSnapshotsRequest>
 
     private boolean ignoreUnavailable;
 
+    private boolean verbose = true;
+
     public GetSnapshotsRequest() {
     }
 
@@ -123,19 +126,44 @@ public GetSnapshotsRequest ignoreUnavailable(boolean ignoreUnavailable) {
         this.ignoreUnavailable = ignoreUnavailable;
         return this;
     }
+
     /**
      * @return Whether snapshots should be ignored when unavailable (corrupt or temporarily not fetchable)
      */
     public boolean ignoreUnavailable() {
         return ignoreUnavailable;
     }
 
+    /**
+     * Set to {@code false} to only show the snapshot names and the indices they contain.
+     * This is useful when the snapshots belong to a cloud-based repository where each
+     * blob read is a concern (cost wise and performance wise), as the snapshot names and
+     * indices they contain can be retrieved from a single index blob in the repository,
+     * whereas the rest of the information requires reading a snapshot metadata file for
+     * each snapshot requested.  Defaults to {@code true}, which returns all information
+     * about each requested snapshot.
+     */
+    public GetSnapshotsRequest verbose(boolean verbose) {
+        this.verbose = verbose;
+        return this;
+    }
+
+    /**
+     * Returns whether the request will return a verbose response.
+     */
+    public boolean verbose() {
+        return verbose;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         repository = in.readString();
         snapshots = in.readStringArray();
         ignoreUnavailable = in.readBoolean();
+        if (in.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            verbose = in.readBoolean();
+        }
     }
 
     @Override
@@ -144,5 +172,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(repository);
         out.writeStringArray(snapshots);
         out.writeBoolean(ignoreUnavailable);
+        if (out.getVersion().onOrAfter(VERBOSE_INTRODUCED)) {
+            out.writeBoolean(verbose);
+        }
     }
 }

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsRequestBuilder.java

@@ -96,4 +96,18 @@ public GetSnapshotsRequestBuilder setIgnoreUnavailable(boolean ignoreUnavailable
         return this;
     }
 
+    /**
+     * Set to {@code false} to only show the snapshot names and the indices they contain.
+     * This is useful when the snapshots belong to a cloud-based repository where each
+     * blob read is a concern (cost wise and performance wise), as the snapshot names and
+     * indices they contain can be retrieved from a single index blob in the repository,
+     * whereas the rest of the information requires reading a snapshot metadata file for
+     * each snapshot requested.  Defaults to {@code true}, which returns all information
+     * about each requested snapshot.
+     */
+    public GetSnapshotsRequestBuilder setVerbose(boolean verbose) {
+        request.verbose(verbose);
+        return this;
+    }
+
 }

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -30,6 +31,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -39,11 +41,13 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Transport Action for get snapshots operation
@@ -76,31 +80,35 @@ protected ClusterBlockException checkBlock(GetSnapshotsRequest request, ClusterS
     }
 
     @Override
-    protected void masterOperation(final GetSnapshotsRequest request, ClusterState state,
+    protected void masterOperation(final GetSnapshotsRequest request, final ClusterState state,
                                    final ActionListener<GetSnapshotsResponse> listener) {
         try {
             final String repository = request.repository();
-            List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
             final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
-            final List<SnapshotId> currentSnapshotIds = new ArrayList<>();
-            final RepositoryData repositoryData  = snapshotsService.getRepositoryData(repository);
+            final List<SnapshotInfo> currentSnapshots = new ArrayList<>();
             for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
                 SnapshotId snapshotId = snapshotInfo.snapshotId();
                 allSnapshotIds.put(snapshotId.getName(), snapshotId);
-                currentSnapshotIds.add(snapshotId);
+                currentSnapshots.add(snapshotInfo);
             }
+
+            final RepositoryData repositoryData;
             if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
+                repositoryData = snapshotsService.getRepositoryData(repository);
                 for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
                     allSnapshotIds.put(snapshotId.getName(), snapshotId);
                 }
+            } else {
+                repositoryData = null;
             }
+
             final Set<SnapshotId> toResolve = new HashSet<>();
             if (isAllSnapshots(request.snapshots())) {
                 toResolve.addAll(allSnapshotIds.values());
             } else {
                 for (String snapshotOrPattern : request.snapshots()) {
                     if (GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshotOrPattern)) {
-                        toResolve.addAll(currentSnapshotIds);
+                        toResolve.addAll(currentSnapshots.stream().map(SnapshotInfo::snapshotId).collect(Collectors.toList()));
                     } else if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                         if (allSnapshotIds.containsKey(snapshotOrPattern)) {
                             toResolve.add(allSnapshotIds.get(snapshotOrPattern));
@@ -121,9 +129,23 @@ protected void masterOperation(final GetSnapshotsRequest request, ClusterState s
                 }
             }
 
-            snapshotInfoBuilder.addAll(snapshotsService.snapshots(
-                repository, new ArrayList<>(toResolve), repositoryData.getIncompatibleSnapshotIds(), request.ignoreUnavailable()));
-            listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
+            final List<SnapshotInfo> snapshotInfos;
+            if (request.verbose()) {
+                final Set<SnapshotId> incompatibleSnapshots = repositoryData != null ?
+                    new HashSet<>(repositoryData.getIncompatibleSnapshotIds()) : Collections.emptySet();
+                snapshotInfos = snapshotsService.snapshots(repository, new ArrayList<>(toResolve),
+                    incompatibleSnapshots, request.ignoreUnavailable());
+            } else {
+                if (repositoryData != null) {
+                    // want non-current snapshots as well, which are found in the repository data
+                    snapshotInfos = buildSimpleSnapshotInfos(toResolve, repositoryData, currentSnapshots);
+                } else {
+                    // only want current snapshots
+                    snapshotInfos = currentSnapshots.stream().map(SnapshotInfo::basic).collect(Collectors.toList());
+                    CollectionUtil.timSort(snapshotInfos);
+                }
+            }
+            listener.onResponse(new GetSnapshotsResponse(snapshotInfos));
         } catch (Exception e) {
             listener.onFailure(e);
         }
@@ -136,4 +158,32 @@ private boolean isAllSnapshots(String[] snapshots) {
     private boolean isCurrentSnapshotsOnly(String[] snapshots) {
         return (snapshots.length == 1 && GetSnapshotsRequest.CURRENT_SNAPSHOT.equalsIgnoreCase(snapshots[0]));
     }
+
+    private List<SnapshotInfo> buildSimpleSnapshotInfos(final Set<SnapshotId> toResolve,
+                                                        final RepositoryData repositoryData,
+                                                        final List<SnapshotInfo> currentSnapshots) {
+        List<SnapshotInfo> snapshotInfos = new ArrayList<>();
+        for (SnapshotInfo snapshotInfo : currentSnapshots) {
+            if (toResolve.remove(snapshotInfo.snapshotId())) {
+                snapshotInfos.add(snapshotInfo.basic());
+            }
+        }
+        Map<SnapshotId, List<String>> snapshotsToIndices = new HashMap<>();
+        for (IndexId indexId : repositoryData.getIndices().values()) {
+            for (SnapshotId snapshotId : repositoryData.getSnapshots(indexId)) {
+                if (toResolve.contains(snapshotId)) {
+                    snapshotsToIndices.computeIfAbsent(snapshotId, (k) -> new ArrayList<>())
+                                      .add(indexId.getName());
+                }
+            }
+        }
+        for (Map.Entry<SnapshotId, List<String>> entry : snapshotsToIndices.entrySet()) {
+            final List<String> indices = entry.getValue();
+            CollectionUtil.timSort(indices);
+            final SnapshotId snapshotId = entry.getKey();
+            snapshotInfos.add(new SnapshotInfo(snapshotId, indices, repositoryData.getSnapshotState(snapshotId)));
+        }
+        CollectionUtil.timSort(snapshotInfos);
+        return Collections.unmodifiableList(snapshotInfos);
+    }
 }

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -163,16 +163,6 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot
 
         try {
             spawner.spawnNativePluginControllers(environment);
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-                @Override
-                public void run() {
-                    try {
-                        spawner.close();
-                    } catch (IOException e) {
-                        throw new ElasticsearchException("Failed to destroy spawned controllers", e);
-                    }
-                }
-            });
         } catch (IOException e) {
             throw new BootstrapException(e);
         }
@@ -191,7 +181,7 @@ public void run() {
                 @Override
                 public void run() {
                     try {
-                        IOUtils.close(node);
+                        IOUtils.close(node, spawner);
                         LoggerContext context = (LoggerContext) LogManager.getContext(false);
                         Configurator.shutdown(context);
                     } catch (IOException ex) {
@@ -269,7 +259,7 @@ private void start() throws NodeValidationException {
 
     static void stop() throws IOException {
         try {
-            IOUtils.close(INSTANCE.node);
+            IOUtils.close(INSTANCE.node, INSTANCE.spawner);
         } finally {
             INSTANCE.keepAliveLatch.countDown();
         }

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -276,7 +276,11 @@ public ClusterState execute(ClusterState currentState) {
                     for (Index index : closeIndices) {
                         final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                         final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
+                        // Verifies that the current index settings can be updated with the updated dynamic settings.
                         indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
+                        // Now check that we can create the index with the updated settings (dynamic and non-dynamic).
+                        // This step is mandatory since we allow to update non-dynamic settings on closed indices.
+                        indicesService.verifyIndexMetadata(updatedMetaData, updatedMetaData);
                     }
                 } catch (IOException ex) {
                     throw ExceptionsHelper.convertToElastic(ex);

core/src/main/java/org/elasticsearch/index/analysis/PreConfiguredTokenFilter.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.analysis.PreBuiltCacheFactory;
+import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy;
 
 import java.io.IOException;
 import java.util.function.BiFunction;
@@ -36,31 +37,46 @@
  * Provides pre-configured, shared {@link TokenFilter}s.
  */
 public final class PreConfiguredTokenFilter implements AnalysisModule.AnalysisProvider<TokenFilterFactory> {
+    /**
+     * Create a pre-configured token filter that may not vary at all.
+     */
+    public static PreConfiguredTokenFilter singleton(String name, boolean useFilterForMultitermQueries,
+            Function<TokenStream, TokenStream> create) {
+        return new PreConfiguredTokenFilter(name, useFilterForMultitermQueries, CachingStrategy.ONE,
+                (tokenStream, version) -> create.apply(tokenStream));
+    }
+
+    /**
+     * Create a pre-configured token filter that may vary based on the Lucene version.
+     */
+    public static PreConfiguredTokenFilter luceneVersion(String name, boolean useFilterForMultitermQueries,
+            BiFunction<TokenStream, org.apache.lucene.util.Version, TokenStream> create) {
+        return new PreConfiguredTokenFilter(name, useFilterForMultitermQueries, CachingStrategy.LUCENE,
+                (tokenStream, version) -> create.apply(tokenStream, version.luceneVersion));
+    }
+
+    /**
+     * Create a pre-configured token filter that may vary based on the Elasticsearch version.
+     */
+    public static PreConfiguredTokenFilter elasticsearchVersion(String name, boolean useFilterForMultitermQueries,
+            BiFunction<TokenStream, org.elasticsearch.Version, TokenStream> create) {
+        return new PreConfiguredTokenFilter(name, useFilterForMultitermQueries, CachingStrategy.ELASTICSEARCH,
+                (tokenStream, version) -> create.apply(tokenStream, version));
+    }
+
     private final String name;
     private final boolean useFilterForMultitermQueries;
     private final PreBuiltCacheFactory.PreBuiltCache<TokenFilterFactory> cache;
     private final BiFunction<TokenStream, Version, TokenStream> create;
 
-    /**
-     * Standard ctor with all the power.
-     */
-    public PreConfiguredTokenFilter(String name, boolean useFilterForMultitermQueries,
-            PreBuiltCacheFactory.CachingStrategy cachingStrategy, BiFunction<TokenStream, Version, TokenStream> create) {
+    private PreConfiguredTokenFilter(String name, boolean useFilterForMultitermQueries,
+            PreBuiltCacheFactory.CachingStrategy cache, BiFunction<TokenStream, Version, TokenStream> create) {
         this.name = name;
         this.useFilterForMultitermQueries = useFilterForMultitermQueries;
-        cache = PreBuiltCacheFactory.getCache(cachingStrategy);
+        this.cache = PreBuiltCacheFactory.getCache(cache);
         this.create = create;
     }
 
-    /**
-     * Convenience ctor for token streams that don't vary based on version.
-     */
-    public PreConfiguredTokenFilter(String name, boolean useFilterForMultitermQueries,
-            PreBuiltCacheFactory.CachingStrategy cachingStrategy, Function<TokenStream, TokenStream> create) {
-        this(name, useFilterForMultitermQueries, cachingStrategy, (input, version) -> create.apply(input));
-        // TODO why oh why aren't these all CachingStrategy.ONE? They *can't* vary based on version because they don't get it, right?!
-    }
-
     @Override
     public TokenFilterFactory get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
         return getTokenFilterFactory(Version.indexCreated(settings));

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -1515,8 +1515,6 @@ public long getGlobalCheckpoint() {
      */
     public void updateGlobalCheckpointOnReplica(final long globalCheckpoint) {
         verifyReplicationTarget();
-        // we sample the recovery stage before sampling the local checkpoint or we are subject to a race condition in the below assertion
-        final RecoveryState.Stage stage = recoveryState().getStage();
         final SequenceNumbersService seqNoService = getEngine().seqNoService();
         final long localCheckpoint = seqNoService.getLocalCheckpoint();
         if (globalCheckpoint > localCheckpoint) {
@@ -1526,10 +1524,10 @@ public void updateGlobalCheckpointOnReplica(final long globalCheckpoint) {
              * case that the global checkpoint update from the primary is ahead of the local checkpoint on this shard. In this case, we
              * ignore the global checkpoint update. This can happen if we are in the translog stage of recovery. Prior to this, the engine
              * is not opened and this shard will not receive global checkpoint updates, and after this the shard will be contributing to
-             * calculations of the the global checkpoint.
+             * calculations of the the global checkpoint. However, we can not assert that we are in the translog stage of recovery here as
+             * while the global checkpoint update may have emanated from the primary when we were in that state, we could subsequently move
+             * to recovery finalization, or even finished recovery before the update arrives here.
              */
-            assert stage == RecoveryState.Stage.TRANSLOG
-                    : "expected recovery stage [" + RecoveryState.Stage.TRANSLOG + "] but was [" + stage + "]";
             return;
         }
         seqNoService.updateGlobalCheckpointOnReplica(globalCheckpoint);