Add tests for the supported query types. (#35319)

* Add tests for the supported query types.
* Disallow unbounded range queries on keyed JSON fields.
* Make sure MappedFieldType#hasDocValues always returns false.

Author: jtibshirani

rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml

@@ -1340,15 +1340,13 @@ setup:
         index:  json_test
         body:
           mappings:
-            _doc:
-              dynamic: false
-              properties:
-                json:
-                  type: json
+            dynamic: false
+            properties:
+              json:
+                type: json
   - do:
       index:
         index:  json_test
-        type:   _doc
         id:     1
         body:
           json:
@@ -1363,7 +1361,7 @@ setup:
             exists:
               field: json
 
-  - match: { hits.total: 1 }
+  - match: { hits.total.value: 1 }
 
   - do:
       search:
@@ -1373,7 +1371,7 @@ setup:
             exists:
               field: json.key
 
-  - match: { hits.total: 1 }
+  - match: { hits.total.value: 1 }
 
   - do:
       search:
@@ -1383,4 +1381,4 @@ setup:
             exists:
               field: json.nonexistent_key
 
-  - match: { hits.total: 0 }
+  - match: { hits.total.value: 0 }

rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yml

@@ -73,15 +73,13 @@
         index:  test
         body:
           mappings:
-            _doc:
-              properties:
-                headers:
-                  type: json
+            properties:
+              headers:
+                type: json
 
   - do:
       index:
         index:  test
-        type:   _doc
         id:     1
         body:
           headers:
@@ -92,7 +90,6 @@
   - do:
       index:
         index:  test
-        type:   _doc
         id:     2
         body:
           headers:
@@ -108,7 +105,7 @@
             query_string:
               query: "headers:text\\/plain"
 
-  - match: { hits.total: 1 }
+  - match: { hits.total.value: 1 }
   - length: { hits.hits: 1 }
   - match: { hits.hits.0._id: "2" }
 
@@ -120,6 +117,6 @@
             query_string:
               query: "application\\/javascript AND headers.origin:elastic.co"
 
-  - match: { hits.total: 1 }
+  - match: { hits.total.value: 1 }
   - length: { hits.hits: 1 }
   - match: { hits.hits.0._id: "1" }

server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java

@@ -29,6 +29,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -165,6 +166,11 @@ public Builder copyTo(CopyTo copyTo) {
             throw new UnsupportedOperationException("[copy_to] is not supported for [" + CONTENT_TYPE + "] fields.");
         }
 
+        @Override
+        protected boolean defaultDocValues(Version indexCreated) {
+            return false;
+        }
+
         @Override
         public JsonFieldMapper build(BuilderContext context) {
             setupFieldType(context);
@@ -276,6 +282,25 @@ public Query existsQuery(QueryShardContext context) {
             return new PrefixQuery(term);
         }
 
+        @Override
+        public Query rangeQuery(Object lowerTerm,
+                                Object upperTerm,
+                                boolean includeLower,
+                                boolean includeUpper,
+                                QueryShardContext context) {
+
+            // We require range queries to specify both bounds because an unbounded query could incorrectly match
+            // values from other keys. For example, a query on the 'first' key with only a lower bound would become
+            // ("first\0value", null), which would also match the value "second\0value" belonging to the key 'second'.
+            if (lowerTerm == null || upperTerm == null) {
+                throw new IllegalArgumentException("[range] queries on keyed [" + CONTENT_TYPE +
+                    "] fields must include both an upper and a lower bound.");
+            }
+
+            return super.rangeQuery(lowerTerm, upperTerm,
+                includeLower, includeUpper, context);
+        }
+
         @Override
         public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
                                 boolean transpositions) {

server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.JsonFieldMapper.KeyedJsonFieldType;
 import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -42,7 +43,6 @@
 
 import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
 
 public class JsonFieldMapperTests extends ESSingleNodeTestCase {
     private IndexService indexService;
@@ -417,7 +417,8 @@ public void testNullValues() throws Exception {
         assertEquals(new BytesRef("key\0placeholder"), prefixedOtherFields[0].binaryValue());
     }
 
-     public void testSplitQueriesOnWhitespace() throws IOException {
+    public void testSplitQueriesOnWhitespace() throws IOException {
+        MapperService mapperService = indexService.mapperService();
         String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
             .startObject("type")
                 .startObject("properties")
@@ -427,13 +428,16 @@ public void testSplitQueriesOnWhitespace() throws IOException {
                     .endObject()
                 .endObject()
             .endObject().endObject());
-        indexService.mapperService().merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
+        mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
 
-        MappedFieldType fieldType = indexService.mapperService().fullName("field");
-        assertThat(fieldType, instanceOf(RootJsonFieldType.class));
+        RootJsonFieldType rootFieldType = (RootJsonFieldType) mapperService.fullName("field");
+        assertThat(rootFieldType.searchAnalyzer(), equalTo(JsonFieldMapper.WHITESPACE_ANALYZER));
+        assertTokenStreamContents(rootFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
+            new String[] {"Hello", "World"});
 
-        RootJsonFieldType ft = (RootJsonFieldType) fieldType;
-        assertThat(ft.searchAnalyzer(), equalTo(JsonFieldMapper.WHITESPACE_ANALYZER));
-        assertTokenStreamContents(ft.searchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"});
+        KeyedJsonFieldType keyedFieldType = (KeyedJsonFieldType) mapperService.fullName("field.key");
+        assertThat(keyedFieldType.searchAnalyzer(), equalTo(JsonFieldMapper.WHITESPACE_ANALYZER));
+        assertTokenStreamContents(keyedFieldType.searchAnalyzer().analyzer().tokenStream("", "Hello World"),
+            new String[] {"Hello", "World"});
     }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeyedJsonFieldTypeTests.java

@@ -20,14 +20,20 @@
 
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.mapper.JsonFieldMapper.KeyedJsonFieldType;
 import org.junit.Before;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class KeyedJsonFieldTypeTests extends FieldTypeTestCase {
 
     @Before
@@ -73,6 +79,22 @@ public void testTermQuery() {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testTermsQuery() {
+        KeyedJsonFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new TermInSetQuery("field",
+            new BytesRef("key\0value1"),
+            new BytesRef("key\0value2"));
+
+        List<String> terms = new ArrayList<>();
+        terms.add("value1");
+        terms.add("value2");
+        Query actual = ft.termsQuery(terms, null);
+
+        assertEquals(expected, actual);
+    }
+
     public void testExistsQuery() {
         KeyedJsonFieldType ft = createDefaultFieldType();
         ft.setName("field");
@@ -81,6 +103,14 @@ public void testExistsQuery() {
         assertEquals(expected, ft.existsQuery(null));
     }
 
+    public void testPrefixQuery() {
+        KeyedJsonFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        Query expected = new PrefixQuery(new Term("field", "key\0val"));
+        assertEquals(expected, ft.prefixQuery("val", MultiTermQuery.CONSTANT_SCORE_REWRITE, null));
+    }
+
     public void testFuzzyQuery() {
         KeyedJsonFieldType ft = createDefaultFieldType();
         ft.setName("field");
@@ -90,6 +120,31 @@ public void testFuzzyQuery() {
         assertEquals("[fuzzy] queries are not currently supported on [json] fields.", e.getMessage());
     }
 
+    public void testRangeQuery() {
+        KeyedJsonFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        TermRangeQuery expected = new TermRangeQuery("field",
+            new BytesRef("key\0lower"),
+            new BytesRef("key\0upper"), false, false);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, null));
+
+        expected = new TermRangeQuery("field",
+            new BytesRef("key\0lower"),
+            new BytesRef("key\0upper"), true, true);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", true, true, null));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            ft.rangeQuery("lower", null, false, false, null));
+        assertEquals("[range] queries on keyed [json] fields must include both an upper and a lower bound.",
+            e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () ->
+            ft.rangeQuery(null, "upper", false, false, null));
+        assertEquals("[range] queries on keyed [json] fields must include both an upper and a lower bound.",
+            e.getMessage());
+    }
+
     public void testRegexpQuery() {
         KeyedJsonFieldType ft = createDefaultFieldType();
         ft.setName("field");

server/src/test/java/org/elasticsearch/index/mapper/RootJsonFieldTypeTests.java

@@ -22,6 +22,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType;
@@ -83,6 +84,21 @@ public void testFuzzyQuery() {
         assertEquals("[fuzzy] queries are not currently supported on [json] fields.", e.getMessage());
     }
 
+    public void testRangeQuery() {
+        RootJsonFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+
+        TermRangeQuery expected = new TermRangeQuery("field",
+            new BytesRef("lower"),
+            new BytesRef("upper"), false, false);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", false, false, null));
+
+        expected = new TermRangeQuery("field",
+            new BytesRef("lower"),
+            new BytesRef("upper"), true, true);
+        assertEquals(expected, ft.rangeQuery("lower", "upper", true, true, null));
+    }
+
     public void testRegexpQuery() {
         RootJsonFieldType ft = createDefaultFieldType();
         ft.setName("field");

server/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java

@@ -67,7 +67,7 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     @Override
     protected MatchQueryBuilder doCreateTestQueryBuilder() {
         String fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME, BOOLEAN_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, DATE_FIELD_NAME);
+            DOUBLE_FIELD_NAME, DATE_FIELD_NAME, JSON_FIELD_NAME);
         Object value;
         if (isTextField(fieldName)) {
             int terms = randomIntBetween(0, 3);
@@ -155,7 +155,8 @@ protected void doAssertLuceneQuery(MatchQueryBuilder queryBuilder, Query query,
         MappedFieldType fieldType = context.fieldMapper(queryBuilder.fieldName());
         if (query instanceof TermQuery && fieldType != null) {
             String queryValue = queryBuilder.value().toString();
-            if (queryBuilder.analyzer() == null || queryBuilder.analyzer().equals("simple")) {
+            if (isTextField(queryBuilder.fieldName())
+                  && (queryBuilder.analyzer() == null || queryBuilder.analyzer().equals("simple"))) {
                 queryValue = queryValue.toLowerCase(Locale.ROOT);
             }
             Query expectedTermQuery = fieldType.termQuery(queryValue, context);
@@ -473,7 +474,7 @@ public void testMaxBooleanClause() {
         query.setAnalyzer(new MockGraphAnalyzer(createGiantGraphMultiTerms()));
         expectThrows(BooleanQuery.TooManyClauses.class, () -> query.parse(Type.PHRASE, STRING_FIELD_NAME, ""));
     }
-    
+
     private static class MockGraphAnalyzer extends Analyzer {
 
         CannedBinaryTokenStream tokenStream;

server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java

@@ -67,7 +67,7 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
     @Override
     protected MultiMatchQueryBuilder doCreateTestQueryBuilder() {
         String fieldName = randomFrom(STRING_FIELD_NAME, INT_FIELD_NAME, DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME,
-                MISSING_FIELD_NAME, MISSING_WILDCARD_FIELD_NAME);
+                MISSING_FIELD_NAME, MISSING_WILDCARD_FIELD_NAME, JSON_FIELD_NAME);
 
         final Object value;
         if (fieldName.equals(STRING_FIELD_NAME)) {

server/src/test/java/org/elasticsearch/index/search/MatchPhraseQueryIT.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
@@ -33,6 +32,7 @@
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 
@@ -55,7 +55,7 @@ public void testZeroTermsQuery() throws ExecutionException, InterruptedException
         List<IndexRequestBuilder> indexRequests = getIndexRequests();
         indexRandom(true, false, indexRequests);
 
-        MatchPhraseQueryBuilder baseQuery = QueryBuilders.matchPhraseQuery("name", "the who")
+        MatchPhraseQueryBuilder baseQuery = matchPhraseQuery("name", "the who")
             .analyzer("standard_stopwords");
 
         MatchPhraseQueryBuilder matchNoneQuery = baseQuery.zeroTermsQuery(ZeroTermsQuery.NONE);
@@ -67,7 +67,6 @@ public void testZeroTermsQuery() throws ExecutionException, InterruptedException
         assertHitCount(matchAllResponse, 2L);
     }
 
-
     private List<IndexRequestBuilder> getIndexRequests() {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         requests.add(client().prepareIndex(INDEX, "band").setSource("name", "the beatles"));