Merge branch 'master' into ccr

jasontedor · jasontedor · commit 3cc1a88f7fee · 2017-10-30T13:16:50.000-04:00
* master: Refactor internal engine [Docs] #26541: add warning regarding the limit on the number of fields that can be queried at once in the multi_match query. [Docs] Fix note in bucket_selector [Docs] Fix indentation of examples (#27168) [Docs] Clarify `span_not` query behavior for non-overlapping matches (#27150) [Docs] Remove first person "I" from getting started (#27155)
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -572,7 +572,11 @@ public CommitStats commitStats() {
         return new CommitStats(getLastCommittedSegmentInfos());
     }
 
-    /** get the sequence number service */
+    /**
+     * The sequence number service for this engine.
+     *
+     * @return the sequence number service
+     */
     public abstract SequenceNumbersService seqNoService();
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java b/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java
@@ -66,7 +66,7 @@ public SequenceNumbersService(
      *
      * @return the next assigned sequence number
      */
-    public long generateSeqNo() {
+    public final long generateSeqNo() {
         return localCheckpointTracker.generateSeqNo();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -316,7 +316,7 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti
                             assert documentFailureMessage != null;
                             throw new IOException(documentFailureMessage);
                         }
-                    }, null, config);
+                    }, null, null, config);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -638,6 +638,7 @@ public long addDocument(final Iterable<? extends IndexableField> doc) throws IOE
                                 }
                             },
                     null,
+                    null,
                     config);
         }
 
diff --git a/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc b/docs/reference/aggregations/pipeline/bucket-selector-aggregation.asciidoc
@@ -6,7 +6,7 @@ in the parent multi-bucket aggregation. The specified metric must be numeric and
 If the script language is `expression` then a numeric return value is permitted. In this case 0.0 will be evaluated as `false`
 and all other values will evaluate to true.
 
-Note: The bucket_selector aggregation, like all pipeline aggregations, executions after all other sibling aggregations. This means that
+NOTE: The bucket_selector aggregation, like all pipeline aggregations, executions after all other sibling aggregations. This means that
 using the bucket_selector aggregation to filter the returned buckets in the response does not save on execution time running the aggregations.
 
 ==== Syntax
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
@@ -13,7 +13,7 @@ Here are a few sample use-cases that Elasticsearch could be used for:
 * You run a price alerting platform which allows price-savvy customers to specify a rule like "I am interested in buying a specific electronic gadget and I want to be notified if the price of gadget falls below $X from any vendor within the next month". In this case you can scrape vendor prices, push them into Elasticsearch and use its reverse-search (Percolator) capability to match price movements against customer queries and eventually push the alerts out to the customer once matches are found.
 * You have analytics/business-intelligence needs and want to quickly investigate, analyze, visualize, and ask ad-hoc questions on a lot of data (think millions or billions of records). In this case, you can use Elasticsearch to store your data and then use Kibana (part of the Elasticsearch/Logstash/Kibana stack) to build custom dashboards that can visualize aspects of your data that are important to you. Additionally, you can use the Elasticsearch aggregations functionality to perform complex business intelligence queries against your data.
 
-For the rest of this tutorial, I will guide you through the process of getting Elasticsearch up and running, taking a peek inside it, and performing basic operations like indexing, searching, and modifying your data. At the end of this tutorial, you should have a good idea of what Elasticsearch is, how it works, and hopefully be inspired to see how you can use it to either build sophisticated search applications or to mine intelligence from your data.
+For the rest of this tutorial, you will be guided through the process of getting Elasticsearch up and running, taking a peek inside it, and performing basic operations like indexing, searching, and modifying your data. At the end of this tutorial, you should have a good idea of what Elasticsearch is, how it works, and hopefully be inspired to see how you can use it to either build sophisticated search applications or to mine intelligence from your data.
 --
 
 == Basic Concepts
@@ -660,7 +660,7 @@ Now that we've gotten a glimpse of the basics, let's try to work on a more reali
 --------------------------------------------------
 // NOTCONSOLE
 
-For the curious, I generated this data from http://www.json-generator.com/[`www.json-generator.com/`] so please ignore the actual values and semantics of the data as these are all randomly generated.
+For the curious, this data was generated using http://www.json-generator.com/[`www.json-generator.com/`], so please ignore the actual values and semantics of the data as these are all randomly generated.
 
 [float]
 === Loading the Sample Dataset
@@ -1284,4 +1284,4 @@ There are many other aggregations capabilities that we won't go into detail here
 
 == Conclusion
 
-Elasticsearch is both a simple and complex product. We've so far learned the basics of what it is, how to look inside of it, and how to work with it using some of the REST APIs. I hope that this tutorial has given you a better understanding of what Elasticsearch is and more importantly, inspired you to further experiment with the rest of its great features!
+Elasticsearch is both a simple and complex product. We've so far learned the basics of what it is, how to look inside of it, and how to work with it using some of the REST APIs. Hopefully this tutorial has given you a better understanding of what Elasticsearch is and more importantly, inspired you to further experiment with the rest of its great features!
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
@@ -76,7 +76,7 @@ GET /_search
         "common": {
             "body": {
                 "query": "this is bonsai cool",
-                    "cutoff_frequency": 0.001
+                "cutoff_frequency": 0.001
             }
         }
     }
@@ -100,8 +100,8 @@ GET /_search
         "common": {
             "body": {
                 "query": "nelly the elephant as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "low_freq_operator": "and"
+                "cutoff_frequency": 0.001,
+                "low_freq_operator": "and"
             }
         }
     }
@@ -200,11 +200,11 @@ GET /_search
         "common": {
             "body": {
                 "query": "nelly the elephant not as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
+                "cutoff_frequency": 0.001,
+                "minimum_should_match": {
+                    "low_freq" : 2,
+                    "high_freq" : 3
+                }
             }
         }
     }
@@ -261,11 +261,11 @@ GET /_search
         "common": {
             "body": {
                 "query": "how not to be",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
+                "cutoff_frequency": 0.001,
+                "minimum_should_match": {
+                    "low_freq" : 2,
+                    "high_freq" : 3
+                }
             }
         }
     }
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -58,6 +58,8 @@ GET /_search
 
 <1> The `subject` field is three times as important as the `message` field.
 
+WARNING: There is a limit of no more than 1024 fields being queried at once.
+
 [[multi-match-types]]
 [float]
 ==== Types of `multi_match` query:
diff --git a/docs/reference/query-dsl/span-not-query.asciidoc b/docs/reference/query-dsl/span-not-query.asciidoc
@@ -1,7 +1,9 @@
 [[query-dsl-span-not-query]]
 === Span Not Query
 
-Removes matches which overlap with another span query. The span not
+Removes matches which overlap with another span query or which are
+within x tokens before (controlled by the parameter `pre`) or y tokens
+after (controled by the parameter `post`) another SpanQuery. The span not
 query maps to Lucene `SpanNotQuery`. Here is an example:
 
 [source,js]
@@ -39,7 +41,7 @@ In the above example all documents with the term hoya are filtered except the on
 Other top level options:
 
 [horizontal]
-`pre`::     If set the amount of tokens before the include span can't have overlap with the exclude span.
-`post`::    If set the amount of tokens after the include span can't have overlap with the exclude span.
+`pre`::     If set the amount of tokens before the include span can't have overlap with the exclude span. Defaults to 0.
+`post`::    If set the amount of tokens after the include span can't have overlap with the exclude span. Defaults to 0.
 `dist`::    If set the amount of tokens from within the include span can't have overlap with the exclude span. Equivalent
             of setting both `pre` and `post`.
diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.engine;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperForType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.RootObjectMapper;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.indices.mapper.MapperRegistry;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.index.mapper.SourceToParse.source;
+
+public class TranslogHandler implements EngineConfig.TranslogRecoveryRunner {
+
+    private final MapperService mapperService;
+    public Mapping mappingUpdate = null;
+    private final Map<String, Mapping> recoveredTypes = new HashMap<>();
+
+    private final AtomicLong appliedOperations = new AtomicLong();
+
+    long appliedOperations() {
+        return appliedOperations.get();
+    }
+
+    public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings indexSettings) {
+        NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
+        IndexAnalyzers indexAnalyzers =
+                new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, emptyMap(), emptyMap());
+        SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap());
+        MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry();
+        mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
+                () -> null);
+    }
+
+    private DocumentMapperForType docMapper(String type) {
+        RootObjectMapper.Builder rootBuilder = new RootObjectMapper.Builder(type);
+        DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService);
+        return new DocumentMapperForType(b.build(mapperService), mappingUpdate);
+    }
+
+    private void applyOperation(Engine engine, Engine.Operation operation) throws IOException {
+        switch (operation.operationType()) {
+            case INDEX:
+                Engine.Index engineIndex = (Engine.Index) operation;
+                Mapping update = engineIndex.parsedDoc().dynamicMappingsUpdate();
+                if (engineIndex.parsedDoc().dynamicMappingsUpdate() != null) {
+                    recoveredTypes.compute(engineIndex.type(), (k, mapping) -> mapping == null ? update : mapping.merge(update, false));
+                }
+                engine.index(engineIndex);
+                break;
+            case DELETE:
+                engine.delete((Engine.Delete) operation);
+                break;
+            case NO_OP:
+                engine.noOp((Engine.NoOp) operation);
+                break;
+            default:
+                throw new IllegalStateException("No operation defined for [" + operation + "]");
+        }
+    }
+
+    /**
+     * Returns the recovered types modifying the mapping during the recovery
+     */
+    public Map<String, Mapping> getRecoveredTypes() {
+        return recoveredTypes;
+    }
+
+    @Override
+    public int run(Engine engine, Translog.Snapshot snapshot) throws IOException {
+        int opsRecovered = 0;
+        Translog.Operation operation;
+        while ((operation = snapshot.next()) != null) {
+            applyOperation(engine, convertToEngineOp(operation, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY));
+            opsRecovered++;
+            appliedOperations.incrementAndGet();
+        }
+        return opsRecovered;
+    }
+
+    private Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.Operation.Origin origin) {
+        switch (operation.opType()) {
+            case INDEX:
+                final Translog.Index index = (Translog.Index) operation;
+                final String indexName = mapperService.index().getName();
+                final Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()),
+                        mapperService.getIndexSettings().getIndexVersionCreated(),
+                        source(indexName, index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source()))
+                                .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(),
+                        index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin,
+                        index.getAutoGeneratedIdTimestamp(), true);
+                return engineIndex;
+            case DELETE:
+                final Translog.Delete delete = (Translog.Delete) operation;
+                final Engine.Delete engineDelete = new Engine.Delete(delete.type(), delete.id(), delete.uid(), delete.seqNo(),
+                        delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(),
+                        origin, System.nanoTime());
+                return engineDelete;
+            case NO_OP:
+                final Translog.NoOp noOp = (Translog.NoOp) operation;
+                final Engine.NoOp engineNoOp =
+                        new Engine.NoOp(noOp.seqNo(), noOp.primaryTerm(), origin, System.nanoTime(), noOp.reason());
+                return engineNoOp;
+            default:
+                throw new IllegalStateException("No operation defined for [" + operation + "]");
+        }
+    }
+
+}

Original file line number	Diff line number	Diff line change
`@@ -66,7 +66,7 @@ public SequenceNumbersService(`
`66`	`66`	`*`
`67`	`67`	`* @return the next assigned sequence number`
`68`	`68`	`*/`
`69`		`- public long generateSeqNo() {`
	`69`	`+ public final long generateSeqNo() {`
`70`	`70`	`return localCheckpointTracker.generateSeqNo();`
`71`	`71`	`}`
`72`	`72`
Original file line number	Diff line number	Diff line change
`@@ -316,7 +316,7 @@ public long addDocument(Iterable<? extends IndexableField> doc) throws IOExcepti`
`316`	`316`	`assert documentFailureMessage != null;`
`317`	`317`	`throw new IOException(documentFailureMessage);`
`318`	`318`	`}`
`319`		`- }, null, config);`
	`319`	`+ }, null, null, config);`
`320`	`320`	`}`
`321`	`321`	`}`
`322`	`322`
Original file line number	Diff line number	Diff line change
`@@ -638,6 +638,7 @@ public long addDocument(final Iterable<? extends IndexableField> doc) throws IOE`
`638`	`638`	`}`
`639`	`639`	`},`
`640`	`640`	`null,`
	`641`	`+ null,`
`641`	`642`	`config);`
`642`	`643`	`}`
`643`	`644`
Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ GET /_search`
`76`	`76`	`"common": {`
`77`	`77`	`"body": {`
`78`	`78`	`"query": "this is bonsai cool",`
`79`		`- "cutoff_frequency": 0.001`
	`79`	`+ "cutoff_frequency": 0.001`
`80`	`80`	`}`
`81`	`81`	`}`
`82`	`82`	`}`
`@@ -100,8 +100,8 @@ GET /_search`
`100`	`100`	`"common": {`
`101`	`101`	`"body": {`
`102`	`102`	`"query": "nelly the elephant as a cartoon",`
`103`		`- "cutoff_frequency": 0.001,`
`104`		`- "low_freq_operator": "and"`
	`103`	`+ "cutoff_frequency": 0.001,`
	`104`	`+ "low_freq_operator": "and"`
`105`	`105`	`}`
`106`	`106`	`}`
`107`	`107`	`}`
`@@ -200,11 +200,11 @@ GET /_search`
`200`	`200`	`"common": {`
`201`	`201`	`"body": {`
`202`	`202`	`"query": "nelly the elephant not as a cartoon",`
`203`		`- "cutoff_frequency": 0.001,`
`204`		`- "minimum_should_match": {`
`205`		`- "low_freq" : 2,`
`206`		`- "high_freq" : 3`
`207`		`- }`
	`203`	`+ "cutoff_frequency": 0.001,`
	`204`	`+ "minimum_should_match": {`
	`205`	`+ "low_freq" : 2,`
	`206`	`+ "high_freq" : 3`
	`207`	`+ }`
`208`	`208`	`}`
`209`	`209`	`}`
`210`	`210`	`}`
`@@ -261,11 +261,11 @@ GET /_search`
`261`	`261`	`"common": {`
`262`	`262`	`"body": {`
`263`	`263`	`"query": "how not to be",`
`264`		`- "cutoff_frequency": 0.001,`
`265`		`- "minimum_should_match": {`
`266`		`- "low_freq" : 2,`
`267`		`- "high_freq" : 3`
`268`		`- }`
	`264`	`+ "cutoff_frequency": 0.001,`
	`265`	`+ "minimum_should_match": {`
	`266`	`+ "low_freq" : 2,`
	`267`	`+ "high_freq" : 3`
	`268`	`+ }`
`269`	`269`	`}`
`270`	`270`	`}`
`271`	`271`	`}`