From 242d4b84af0675a5dac80f0aa59f844e2d9452b5 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Wed, 26 Sep 2018 10:22:02 -0700 Subject: [PATCH 1/3] Make sure 'ignored' and 'routing' field types inherit from StringFieldType. This allows them to support additional query types like prefix and wildcard. --- .../index/mapper/IgnoredFieldMapper.java | 2 +- .../index/mapper/RoutingFieldMapper.java | 2 +- .../index/mapper/IgnoredFieldTypeTests.java | 14 ++++++++++++++ .../index/mapper/RoutingFieldTypeTests.java | 16 ++++++++++++++-- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index 7a3a9a8f2ae5a..dc6aee5b9dc24 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -85,7 +85,7 @@ public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext c } } - public static final class IgnoredFieldType extends TermBasedFieldType { + public static final class IgnoredFieldType extends StringFieldType { public IgnoredFieldType() { } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 5411c4604acf6..6a171b767c8b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -108,7 +108,7 @@ public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext c } } - static final class RoutingFieldType extends TermBasedFieldType { + static final class RoutingFieldType extends StringFieldType { RoutingFieldType() { } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java index 4035383893d81..dd80cd1fac764 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -19,6 +19,12 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; + public class IgnoredFieldTypeTests extends FieldTypeTestCase { @Override @@ -26,4 +32,12 @@ protected MappedFieldType createDefaultFieldType() { return new IgnoredFieldMapper.IgnoredFieldType(); } + public void testWildcardQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new WildcardQuery(new Term("field", new BytesRef("foo*"))); + assertEquals(expected, ft.wildcardQuery("foo*", null, null)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java index d64c4c5b0cf28..63d3a50c0e4b3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java @@ -18,12 +18,24 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; public class RoutingFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { return new RoutingFieldMapper.RoutingFieldType(); } + + public void testWildcardQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new WildcardQuery(new Term("field", new BytesRef("foo*"))); + assertEquals(expected, ft.wildcardQuery("foo*", null, null)); + } } From f8808cc9141b14a03444e7baf79f605e84a138dd Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Wed, 26 Sep 2018 14:23:15 -0700 Subject: [PATCH 2/3] Add tests for prefix and regexp queries. --- .../index/mapper/IgnoredFieldTypeTests.java | 20 +++++++++++++++++++ .../index/mapper/RoutingFieldTypeTests.java | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java index dd80cd1fac764..e0cd3b1d153fd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -21,7 +21,9 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; @@ -32,6 +34,24 @@ protected MappedFieldType createDefaultFieldType() { return new IgnoredFieldMapper.IgnoredFieldType(); } + public void testPrefixQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new PrefixQuery(new Term("field", new BytesRef("foo*"))); + assertEquals(expected, ft.prefixQuery("foo*", null, null)); + } + + public void testRegexpQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new RegexpQuery(new Term("field", new BytesRef("foo?"))); + assertEquals(expected, ft.regexpQuery("foo?", 0, 10, null, null)); + } + public void testWildcardQuery() { MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java index 63d3a50c0e4b3..6f68d28c0176a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java @@ -20,7 +20,9 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; @@ -30,6 +32,24 @@ protected MappedFieldType createDefaultFieldType() { return new RoutingFieldMapper.RoutingFieldType(); } + public void testPrefixQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new PrefixQuery(new Term("field", new BytesRef("foo*"))); + assertEquals(expected, ft.prefixQuery("foo*", null, null)); + } + + public void testRegexpQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + Query expected = new RegexpQuery(new Term("field", new BytesRef("foo?"))); + assertEquals(expected, ft.regexpQuery("foo?", 0, 10, null, null)); + } + public void testWildcardQuery() { MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); From 0c9fefe5bce4e5151d50cb4e72149f1fd36cc8cb Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 27 Sep 2018 14:16:57 -0700 Subject: [PATCH 3/3] Support prefix and regexp queries on _index fields. --- .../index/mapper/IndexFieldMapper.java | 34 ++++++++++++- .../index/mapper/IndexFieldTypeTests.java | 48 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 456805e64160e..7e8ac563cacc1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; public class IndexFieldMapper extends MetadataFieldMapper { @@ -151,14 +152,43 @@ public Query termsQuery(List values, QueryShardContext context) { + " vs. " + values); } + @Override + public Query prefixQuery(String value, + @Nullable MultiTermQuery.RewriteMethod method, + QueryShardContext context) { + String indexName = context.getFullyQualifiedIndex().getName(); + if (indexName.startsWith(value)) { + return Queries.newMatchAllQuery(); + } else { + return Queries.newMatchNoDocsQuery("The index [" + indexName + + "] doesn't match the provided prefix [" + value + "]."); + } + } + + @Override + public Query regexpQuery(String value, int flags, int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, QueryShardContext context) { + String indexName = context.getFullyQualifiedIndex().getName(); + Pattern pattern = Regex.compile(value, Regex.flagsToString(flags)); + + if (pattern.matcher(indexName).matches()) { + return Queries.newMatchAllQuery(); + } else { + return Queries.newMatchNoDocsQuery("The index [" + indexName + + "] doesn't match the provided pattern [" + value + "]."); + } + } + @Override public Query wildcardQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) { - if (isSameIndex(value, context.getFullyQualifiedIndex().getName())) { + String indexName = context.getFullyQualifiedIndex().getName(); + if (isSameIndex(value, indexName)) { return Queries.newMatchAllQuery(); } else { - return Queries.newMatchNoDocsQuery("Index didn't match. Index queried: " + context.index().getName() + " vs. " + value); + return Queries.newMatchNoDocsQuery("The index [" + indexName + + "] doesn't match the provided pattern [" + value + "]."); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index ecea620f11cf1..82f0edf24f4da 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -18,12 +18,56 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.index.mapper.IndexFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.query.QueryShardContext; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class IndexFieldTypeTests extends FieldTypeTestCase { + @Override protected MappedFieldType createDefaultFieldType() { return new IndexFieldMapper.IndexFieldType(); } + + public void testPrefixQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("ind", null, createContext())); + assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("other_ind", null, createContext())); + } + + public void testRegexpQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + assertEquals(new MatchAllDocsQuery(), ft.regexpQuery("ind.x", 0, 10, null, createContext())); + assertEquals(new MatchNoDocsQuery(), ft.regexpQuery("ind?x", 0, 10, null, createContext())); + } + + public void testWildcardQuery() { + MappedFieldType ft = createDefaultFieldType(); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + + assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("ind*x", null, createContext())); + assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("other_ind*x", null, createContext())); + } + + private QueryShardContext createContext() { + QueryShardContext context = mock(QueryShardContext.class); + + Index index = new Index("index", "123"); + when(context.getFullyQualifiedIndex()).thenReturn(index); + when(context.index()).thenReturn(index); + + return context; + } }