Merge branch 'master' into split_shards

Author: s1monw

CONTRIBUTING.md

@@ -38,6 +38,11 @@ If you have a bugfix or new feature that you would like to contribute to Elastic
 
 We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code.
 
+Note that it is unlikely the project will merge refactors for the sake of refactoring. These
+types of pull requests have a high cost to maintainers in reviewing and testing with little
+to no tangible benefit. This especially includes changes generated by tools. For example,
+converting all generic interface instances to use the diamond operator. 
+
 The process for contributing to any of the [Elastic repositories](https://github.com/elastic/) is similar. Details for individual projects can be found below.
 
 ### Fork and clone the repository

core/src/main/java/org/elasticsearch/Version.java

@@ -98,6 +98,8 @@ public class Version implements Comparable<Version> {
     public static final Version V_5_6_2 = new Version(V_5_6_2_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_5_6_3_ID = 5060399;
     public static final Version V_5_6_3 = new Version(V_5_6_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
+    public static final int V_5_6_4_ID = 5060499;
+    public static final Version V_5_6_4 = new Version(V_5_6_4_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
     public static final int V_6_0_0_alpha1_ID = 6000001;
     public static final Version V_6_0_0_alpha1 =
             new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
@@ -153,6 +155,8 @@ public static Version fromId(int id) {
                 return V_6_0_0_alpha2;
             case V_6_0_0_alpha1_ID:
                 return V_6_0_0_alpha1;
+            case V_5_6_4_ID:
+                return V_5_6_4;
             case V_5_6_3_ID:
                 return V_5_6_3;
             case V_5_6_2_ID:

core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java

@@ -407,11 +407,10 @@ private ThreadContextStruct putHeaders(Map<String, String> headers) {
             if (headers.isEmpty()) {
                 return this;
             } else {
-                final Map<String, String> newHeaders = new HashMap<>();
+                final Map<String, String> newHeaders = new HashMap<>(this.requestHeaders);
                 for (Map.Entry<String, String> entry : headers.entrySet()) {
                     putSingleHeader(entry.getKey(), entry.getValue(), newHeaders);
                 }
-                newHeaders.putAll(this.requestHeaders);
                 return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders, isSystemContext);
             }
         }

core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
@@ -58,9 +59,10 @@ ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException
 
         final Mapping mapping = docMapper.mapping();
         final ParseContext.InternalParseContext context;
-        try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), source.source())) {
-            context = new ParseContext.InternalParseContext(indexSettings.getSettings(),
-                    docMapperParser, docMapper, source, parser);
+        final XContentType xContentType = source.getXContentType();
+
+        try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), source.source(), xContentType)) {
+            context = new ParseContext.InternalParseContext(indexSettings.getSettings(), docMapperParser, docMapper, source, parser);
             validateStart(parser);
             internalParseDocument(mapping, context, parser);
             validateEnd(parser);
@@ -74,8 +76,7 @@ ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException
 
         reverseOrder(context);
 
-        ParsedDocument doc = parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers()));
-        return doc;
+        return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers()));
     }
 
     private static void internalParseDocument(Mapping mapping, ParseContext.InternalParseContext context, XContentParser parser) throws IOException {
@@ -89,7 +90,7 @@ private static void internalParseDocument(Mapping mapping, ParseContext.Internal
             // entire type is disabled
             parser.skipChildren();
         } else if (emptyDoc == false) {
-            parseObjectOrNested(context, mapping.root, true);
+            parseObjectOrNested(context, mapping.root);
         }
 
         for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
@@ -338,7 +339,7 @@ private static ObjectMapper createUpdate(ObjectMapper parent, String[] nameParts
         return parent.mappingUpdate(mapper);
     }
 
-    static void parseObjectOrNested(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
+    static void parseObjectOrNested(ParseContext context, ObjectMapper mapper) throws IOException {
         if (mapper.isEnabled() == false) {
             context.parser().skipChildren();
             return;
@@ -467,7 +468,7 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map
 
     private static void parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
         if (mapper instanceof ObjectMapper) {
-            parseObjectOrNested(context, (ObjectMapper) mapper, false);
+            parseObjectOrNested(context, (ObjectMapper) mapper);
         } else {
             FieldMapper fieldMapper = (FieldMapper)mapper;
             Mapper update = fieldMapper.parse(context);
@@ -481,14 +482,13 @@ private static void parseObjectOrField(ParseContext context, Mapper mapper) thro
     private static void parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName) throws IOException {
         assert currentFieldName != null;
 
-        Mapper objectMapper = getMapper(mapper, currentFieldName);
+        final String[] paths = splitAndValidatePath(currentFieldName);
+        Mapper objectMapper = getMapper(mapper, currentFieldName, paths);
         if (objectMapper != null) {
             context.path().add(currentFieldName);
             parseObjectOrField(context, objectMapper);
             context.path().remove();
         } else {
-
-            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
             ObjectMapper parentMapper = parentMapperTuple.v2();
@@ -518,7 +518,9 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
 
     private static void parseArray(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
         String arrayFieldName = lastFieldName;
-        Mapper mapper = getMapper(parentMapper, lastFieldName);
+
+        final String[] paths = splitAndValidatePath(arrayFieldName);
+        Mapper mapper = getMapper(parentMapper, lastFieldName, paths);
         if (mapper != null) {
             // There is a concrete mapper for this field already. Need to check if the mapper
             // expects an array, if so we pass the context straight to the mapper and if not
@@ -529,8 +531,6 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
                 parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
             }
         } else {
-
-            final String[] paths = splitAndValidatePath(arrayFieldName);
             arrayFieldName = paths[paths.length - 1];
             lastFieldName = arrayFieldName;
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
@@ -589,12 +589,12 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa
         if (currentFieldName == null) {
             throw new MapperParsingException("object mapping [" + parentMapper.name() + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]");
         }
-        Mapper mapper = getMapper(parentMapper, currentFieldName);
+
+        final String[] paths = splitAndValidatePath(currentFieldName);
+        Mapper mapper = getMapper(parentMapper, currentFieldName, paths);
         if (mapper != null) {
             parseObjectOrField(context, mapper);
         } else {
-
-            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
             parentMapper = parentMapperTuple.v2();
@@ -607,7 +607,7 @@ private static void parseValue(final ParseContext context, ObjectMapper parentMa
 
     private static void parseNullValue(ParseContext context, ObjectMapper parentMapper, String lastFieldName) throws IOException {
         // we can only handle null values if we have mappings for them
-        Mapper mapper = getMapper(parentMapper, lastFieldName);
+        Mapper mapper = getMapper(parentMapper, lastFieldName, splitAndValidatePath(lastFieldName));
         if (mapper != null) {
             // TODO: passing null to an object seems bogus?
             parseObjectOrField(context, mapper);
@@ -893,15 +893,15 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
                             break;
                         case FALSE:
                            // Should not dynamically create any more mappers so return the last mapper
-                        return new Tuple<Integer, ObjectMapper>(pathsAdded, parent);
+                        return new Tuple<>(pathsAdded, parent);
 
                     }
                 }
                 context.path().add(paths[i]);
                 pathsAdded++;
                 parent = mapper;
             }
-        return new Tuple<Integer, ObjectMapper>(pathsAdded, mapper);
+        return new Tuple<>(pathsAdded, mapper);
     }
 
     // find what the dynamic setting is given the current parse context and parent
@@ -929,8 +929,7 @@ private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper,
     }
 
     // looks up a child mapper, but takes into account field names that expand to objects
-    static Mapper getMapper(ObjectMapper objectMapper, String fieldName) {
-        String[] subfields = splitAndValidatePath(fieldName);
+    private static Mapper getMapper(ObjectMapper objectMapper, String fieldName, String[] subfields) {
         for (int i = 0; i < subfields.length - 1; ++i) {
             Mapper mapper = objectMapper.getMapper(subfields[i]);
             if (mapper == null || (mapper instanceof ObjectMapper) == false) {

core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java

@@ -54,8 +54,8 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
     public static final int DEFAULT_MAX_EXPANSIONS = FuzzyQuery.defaultMaxExpansions;
 
     /** Default as to whether transpositions should be treated as a primitive edit operation,
-     * instead of classic Levenshtein algorithm. Defaults to false. */
-    public static final boolean DEFAULT_TRANSPOSITIONS = false;
+     * instead of classic Levenshtein algorithm. Defaults to true. */
+    public static final boolean DEFAULT_TRANSPOSITIONS = FuzzyQuery.defaultTranspositions;
 
     private static final ParseField TERM_FIELD = new ParseField("term");
     private static final ParseField VALUE_FIELD = new ParseField("value");
@@ -74,7 +74,6 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
 
     private int maxExpansions = DEFAULT_MAX_EXPANSIONS;
 
-    //LUCENE 4 UPGRADE  we need a testcase for this + documentation
     private boolean transpositions = DEFAULT_TRANSPOSITIONS;
 
     private String rewrite;

core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java

@@ -30,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -294,13 +295,13 @@ public static ElasticsearchException newConfigurationException(String processorT
         return exception;
     }
 
-    public static List<Processor> readProcessorConfigs(List<Map<String, Map<String, Object>>> processorConfigs,
+    public static List<Processor> readProcessorConfigs(List<Map<String, Object>> processorConfigs,
                                                        Map<String, Processor.Factory> processorFactories) throws Exception {
         Exception exception = null;
         List<Processor> processors = new ArrayList<>();
         if (processorConfigs != null) {
-            for (Map<String, Map<String, Object>> processorConfigWithKey : processorConfigs) {
-                for (Map.Entry<String, Map<String, Object>> entry : processorConfigWithKey.entrySet()) {
+            for (Map<String, Object> processorConfigWithKey : processorConfigs) {
+                for (Map.Entry<String, Object> entry : processorConfigWithKey.entrySet()) {
                     try {
                         processors.add(readProcessor(processorFactories, entry.getKey(), entry.getValue()));
                     } catch (Exception e) {
@@ -353,13 +354,28 @@ private static void addHeadersToException(ElasticsearchException exception, Stri
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public static Processor readProcessor(Map<String, Processor.Factory> processorFactories,
+                                          String type, Object config) throws Exception {
+        if (config instanceof Map) {
+            return readProcessor(processorFactories, type, (Map<String, Object>) config);
+        } else if (config instanceof String && "script".equals(type)) {
+            Map<String, Object> normalizedScript = new HashMap<>(1);
+            normalizedScript.put(ScriptType.INLINE.getName(), config);
+            return readProcessor(processorFactories, type, normalizedScript);
+        } else {
+            throw newConfigurationException(type, null, null,
+                "property isn't a map, but of type [" + config.getClass().getName() + "]");
+        }
+    }
+
     public static Processor readProcessor(Map<String, Processor.Factory> processorFactories,
                                            String type, Map<String, Object> config) throws Exception {
         String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
         Processor.Factory factory = processorFactories.get(type);
         if (factory != null) {
             boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
-            List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
+            List<Map<String, Object>> onFailureProcessorConfigs =
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorFactories);

core/src/main/java/org/elasticsearch/ingest/Pipeline.java

@@ -118,9 +118,9 @@ public static final class Factory {
         public Pipeline create(String id, Map<String, Object> config, Map<String, Processor.Factory> processorFactories) throws Exception {
             String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
             Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null);
-            List<Map<String, Map<String, Object>>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
+            List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
             List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorFactories);
-            List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
+            List<Map<String, Object>> onFailureProcessorConfigs =
                     ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
             List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(onFailureProcessorConfigs, processorFactories);
             if (config.isEmpty() == false) {