From 9fbe3f778f68d51eae37af22fdf4163b11915184 Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Wed, 26 Jan 2022 22:13:43 +0800 Subject: [PATCH 1/8] FilterPathBasedFilter support match fieldname with dot --- .../subphase/FetchSourcePhaseBenchmark.java | 2 +- .../xcontent/FilterContentBenchmark.java | 15 +- .../xcontent/XContentParserConfiguration.java | 58 ++++- .../support/filtering/FilterPath.java | 34 ++- .../filtering/FilterPathBasedFilter.java | 15 +- .../AbstractXContentFilteringTestCase.java | 229 ++++++++++++++++-- .../support/filtering/FilterPathTests.java | 93 ++++--- .../cluster/metadata/IndexAbstraction.java | 3 +- .../cluster/routing/IndexRouting.java | 103 ++++---- .../common/xcontent/XContentHelper.java | 4 +- .../cluster/routing/IndexRoutingTests.java | 43 ++-- 11 files changed, 461 insertions(+), 138 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java index 5c768a7dcb1ef..690483bc8d81d 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -66,7 +66,7 @@ public void setup() throws IOException { ); includesSet = Set.of(fetchContext.includes()); excludesSet = Set.of(fetchContext.excludes()); - parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet); + parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false); } private BytesReference read300BytesExample() throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java index 6f4e926dbe969..02b296fecb4b8 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java @@ -61,6 +61,7 @@ public class FilterContentBenchmark { private BytesReference source; private XContentParserConfiguration parserConfig; private Set filters; + private XContentParserConfiguration parserConfigMatchDotsInFieldNames; @Setup public void setup() throws IOException { @@ -72,7 +73,8 @@ public void setup() throws IOException { }; source = readSource(sourceFile); filters = buildFilters(); - parserConfig = buildParseConfig(); + parserConfig = buildParseConfig(false); + parserConfigMatchDotsInFieldNames = buildParseConfig(true); } private Set buildFilters() { @@ -105,9 +107,14 @@ public BytesReference filterWithParserConfigCreated() throws IOException { return filter(this.parserConfig); } + @Benchmark + public BytesReference filterWithParserConfigCreatedMatchDotsInFieldNames() throws IOException { + return filter(this.parserConfigMatchDotsInFieldNames); + } + @Benchmark public BytesReference filterWithNewParserConfig() throws IOException { - XContentParserConfiguration contentParserConfiguration = buildParseConfig(); + XContentParserConfiguration contentParserConfiguration = buildParseConfig(false); return filter(contentParserConfiguration); } @@ -152,7 +159,7 @@ public BytesReference filterWithBuilder() throws IOException { } } - private XContentParserConfiguration buildParseConfig() { + private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNames) { Set includes; Set excludes; if (inclusive) { @@ -162,7 +169,7 @@ private XContentParserConfiguration buildParseConfig() { includes = null; excludes = filters; } - return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes); + return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames); } private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java index 21765a7e3a54d..66b4a62a7095d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java @@ -32,7 +32,8 @@ public class XContentParserConfiguration { DeprecationHandler.THROW_UNSUPPORTED_OPERATION, RestApiVersion.current(), null, - null + null, + false ); final NamedXContentRegistry registry; @@ -40,26 +41,36 @@ public class XContentParserConfiguration { final RestApiVersion restApiVersion; final FilterPath[] includes; final FilterPath[] excludes; + final boolean filtersMatchFieldNamesWithDots; private XContentParserConfiguration( NamedXContentRegistry registry, DeprecationHandler deprecationHandler, RestApiVersion restApiVersion, FilterPath[] includes, - FilterPath[] excludes + FilterPath[] excludes, + boolean filtersMatchFieldNamesWithDots ) { this.registry = registry; this.deprecationHandler = deprecationHandler; this.restApiVersion = restApiVersion; this.includes = includes; this.excludes = excludes; + this.filtersMatchFieldNamesWithDots = filtersMatchFieldNamesWithDots; } /** * Replace the registry backing {@link XContentParser#namedObject}. */ public XContentParserConfiguration withRegistry(NamedXContentRegistry registry) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public NamedXContentRegistry registry() { @@ -71,7 +82,14 @@ public NamedXContentRegistry registry() { * a deprecated field. */ public XContentParserConfiguration withDeprecationHandler(DeprecationHandler deprecationHandler) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public DeprecationHandler deprecationHandler() { @@ -83,7 +101,14 @@ public DeprecationHandler deprecationHandler() { * {@link RestApiVersion}. */ public XContentParserConfiguration withRestApiVersion(RestApiVersion restApiVersion) { - return new XContentParserConfiguration(registry, deprecationHandler, restApiVersion, includes, excludes); + return new XContentParserConfiguration( + registry, + deprecationHandler, + restApiVersion, + includes, + excludes, + filtersMatchFieldNamesWithDots + ); } public RestApiVersion restApiVersion() { @@ -93,13 +118,18 @@ public RestApiVersion restApiVersion() { /** * Replace the configured filtering. */ - public XContentParserConfiguration withFiltering(Set includeStrings, Set excludeStrings) { + public XContentParserConfiguration withFiltering( + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ) { return new XContentParserConfiguration( registry, deprecationHandler, restApiVersion, FilterPath.compile(includeStrings), - FilterPath.compile(excludeStrings) + FilterPath.compile(excludeStrings), + filtersMatchFieldNamesWithDots ); } @@ -112,10 +142,20 @@ public JsonParser filter(JsonParser parser) { throw new UnsupportedOperationException("double wildcards are not supported in filtered excludes"); } } - filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(excludes, false), true, true); + filtered = new FilteringParserDelegate( + filtered, + new FilterPathBasedFilter(excludes, false, filtersMatchFieldNamesWithDots), + true, + true + ); } if (includes != null) { - filtered = new FilteringParserDelegate(filtered, new FilterPathBasedFilter(includes, true), true, true); + filtered = new FilteringParserDelegate( + filtered, + new FilterPathBasedFilter(includes, true, filtersMatchFieldNamesWithDots), + true, + true + ); } return filtered; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java index 9bb289bfccd47..658ad1eb5ab32 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java @@ -68,14 +68,25 @@ private boolean isFinalNode() { * if current node is a double wildcard node, the node will also add to nextFilters. * @param name the xcontent property name * @param nextFilters nextFilters is a List, used to check the inner property of name + * @param matchFieldNamesWithDots support dot in field name or not * @return true if the name equal a final node, otherwise return false */ - boolean matches(String name, List nextFilters) { + boolean matches(String name, List nextFilters, boolean matchFieldNamesWithDots) { if (nextFilters == null) { return false; } - FilterPath termNode = termsChildren.get(name); + // match dot first + FilterPath termNode; + if (matchFieldNamesWithDots) { + // contains dot and not the first or last char + int dotIndex = name.indexOf('.'); + if ((dotIndex != -1) && (dotIndex != 0) && (dotIndex != name.length() - 1)) { + return matchFieldNamesWithDots(name, dotIndex, nextFilters); + } + } + + termNode = termsChildren.get(name); if (termNode != null) { if (termNode.isFinalNode()) { return true; @@ -102,6 +113,25 @@ boolean matches(String name, List nextFilters) { return false; } + private boolean matchFieldNamesWithDots(String name, int dotIndex, List nextFilters) { + String prefixName = name.substring(0, dotIndex); + String suffixName = name.substring(dotIndex + 1); + List prefixFilterPath = new ArrayList<>(); + boolean prefixMatch = matches(prefixName, prefixFilterPath, true); + // if prefixMatch return true(because prefix is a final FilterPath node) + if (prefixMatch) { + return true; + } + // if has prefixNextFilter, use them to match suffix + for (FilterPath filter : prefixFilterPath) { + boolean matches = filter.matches(suffixName, nextFilters, true); + if (matches) { + return true; + } + } + return false; + } + private static class FilterPathBuilder { private class BuildNode { private final Map children; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java index dc0ca01cc3f3c..ef9411ba2f79f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPathBasedFilter.java @@ -42,16 +42,19 @@ public String toString() { private final boolean inclusive; - public FilterPathBasedFilter(FilterPath[] filters, boolean inclusive) { + private final boolean matchFieldNamesWithDots; + + public FilterPathBasedFilter(FilterPath[] filters, boolean inclusive, boolean matchFieldNamesWithDots) { if (filters == null || filters.length == 0) { throw new IllegalArgumentException("filters cannot be null or empty"); } this.inclusive = inclusive; this.filters = filters; + this.matchFieldNamesWithDots = matchFieldNamesWithDots; } public FilterPathBasedFilter(Set filters, boolean inclusive) { - this(FilterPath.compile(filters), inclusive); + this(FilterPath.compile(filters), inclusive, false); } /** @@ -61,14 +64,18 @@ private TokenFilter evaluate(String name, FilterPath[] filterPaths) { if (filterPaths != null) { List nextFilters = new ArrayList<>(); for (FilterPath filter : filterPaths) { - boolean matches = filter.matches(name, nextFilters); + boolean matches = filter.matches(name, nextFilters, matchFieldNamesWithDots); if (matches) { return MATCHING; } } if (nextFilters.isEmpty() == false) { - return new FilterPathBasedFilter(nextFilters.toArray(new FilterPath[nextFilters.size()]), inclusive); + return new FilterPathBasedFilter( + nextFilters.toArray(new FilterPath[nextFilters.size()]), + inclusive, + matchFieldNamesWithDots + ); } } return NO_MATCHING; diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 3db8a13992ef4..9744cf533010e 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -29,9 +29,207 @@ import static org.hamcrest.Matchers.nullValue; public abstract class AbstractXContentFilteringTestCase extends AbstractFilteringTestCase { + public void testSingleFieldObject() throws IOException { + Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); + Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); + testFilter(expected, sample, singleton("foo.bar"), emptySet()); + testFilter(expected, sample, emptySet(), singleton("foo.baz")); + testFilter(expected, sample, singleton("foo"), singleton("foo.baz")); + + expected = builder -> builder.startObject().endObject(); + testFilter(expected, sample, emptySet(), singleton("foo.bar")); + testFilter(expected, sample, singleton("foo"), singleton("foo.b*")); + } + + public void testDotInIncludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("foo.bar"), + emptySet(), + false + ); + } + + public void testDotInIncludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotInExcludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton("foo.bar"), + false + ); + } + public void testDotInExcludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotInIncludedObjectNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton("foo.bar"), + emptySet(), + false + ); + } + + public void testDotInIncludedObjectNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotInExcludedObjectNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("foo.bar"), + false + ); + } + + public void testDotInExcludedObjectNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotInIncludedFieldNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + emptySet(), + false + ); + } + + public void testDotInIncludedFieldNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + emptySet(), + true + ); + } + + public void testDotInExcludedFieldNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton(randomFrom("foo.*", "foo.*ar")), + false + ); + } + + public void testDotInExcludedFieldNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + emptySet(), + singleton("*.bar"), +// singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + true + ); + } + + public void testDotInIncludedObjectNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton(randomFrom("foo.*", "f*.bar", "foo.*ar")), + emptySet(), + false + ); + } + + public void testDotInIncludedObjectNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + emptySet(), + true + ); + } + + public void testDotInExcludedObjectNamePatternUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + false + ); + } + + public void testDotInExcludedObjectNamePatternConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().startObject("foo.bar").field("baz", "test").endObject().endObject(), + emptySet(), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar")), + true + ); + } + + public void testDotInStarMatchDotsInNamesUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar", "test").endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("f*r"), + emptySet(), + false + ); + } + + public void testDotInStarMatchDotsInNamesConfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar", "test").endObject(), + singleton("f*r"), + emptySet(), + true + ); + } + + @Override protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { - assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes)); + testFilter(expected, sample, includes, excludes, false); + } + + private void testFilter(Builder expected, Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { + assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots)); } protected abstract void assertFilterResult(XContentBuilder expected, XContentBuilder actual); @@ -47,8 +245,9 @@ private XContentBuilder createBuilder() throws IOException { return XContentBuilder.builder(getXContentType().xContent()); } - private XContentBuilder filter(Builder sample, Set includes, Set excludes) throws IOException { - if (randomBoolean()) { + private XContentBuilder filter(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { + if (matchFieldNamesWithDots == false && randomBoolean()) { return filterOnBuilder(sample, includes, excludes); } FilterPath[] excludesFilter = FilterPath.compile(excludes); @@ -58,21 +257,26 @@ private XContentBuilder filter(Builder sample, Set includes, Set * filtering produced weird invalid json. Just field names * and no objects?! Weird. Anyway, we can't use it. */ + assertFalse("can't filter on builder with dotted wildcards in exclude", matchFieldNamesWithDots); return filterOnBuilder(sample, includes, excludes); } - return filterOnParser(sample, includes, excludes); + return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots); } private XContentBuilder filterOnBuilder(Builder sample, Set includes, Set excludes) throws IOException { return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes)); } - private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes) throws IOException { + private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) + throws IOException { try (XContentBuilder builtSample = sample.apply(createBuilder())) { BytesReference sampleBytes = BytesReference.bytes(builtSample); try ( XContentParser parser = getXContentType().xContent() - .createParser(XContentParserConfiguration.EMPTY.withFiltering(includes, excludes), sampleBytes.streamInput()) + .createParser( + XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots), + sampleBytes.streamInput() + ) ) { XContentBuilder result = createBuilder(); if (sampleBytes.get(sampleBytes.length() - 1) == '\n') { @@ -87,19 +291,6 @@ private XContentBuilder filterOnParser(Builder sample, Set includes, Set } } - public void testSingleFieldObject() throws IOException { - final Builder sample = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); - - Builder expected = builder -> builder.startObject().startObject("foo").field("bar", "test").endObject().endObject(); - testFilter(expected, sample, singleton("foo.bar"), emptySet()); - testFilter(expected, sample, emptySet(), singleton("foo.baz")); - testFilter(expected, sample, singleton("foo"), singleton("foo.baz")); - - expected = builder -> builder.startObject().endObject(); - testFilter(expected, sample, emptySet(), singleton("foo.bar")); - testFilter(expected, sample, singleton("foo"), singleton("foo.b*")); - } - static void assertXContentBuilderAsString(final XContentBuilder expected, final XContentBuilder actual) { assertThat(Strings.toString(actual), is(Strings.toString(expected))); } diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java index 8545a497fdb68..716bbce5bdfb0 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java @@ -32,7 +32,7 @@ public void testSimpleFilterPath() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; - assertThat(filterPath.matches("test", nextFilters), is(true)); + assertThat(filterPath.matches("test", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -46,12 +46,12 @@ public void testFilterPathWithSubField() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 1); } @@ -65,18 +65,18 @@ public void testFilterPathWithSubFields() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(false)); + assertThat(filterPath.matches("bar", nextFilters, false), is(false)); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("quz", nextFilters), is(true)); + assertThat(filterPath.matches("quz", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -102,25 +102,25 @@ public void testFilterPathWithEscapedDots() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; assertNotNull(filterPath); - assertThat(filterPath.matches("w", nextFilters), is(false)); + assertThat(filterPath.matches("w", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0", nextFilters), is(false)); + assertThat(filterPath.matches("0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0", nextFilters), is(false)); + assertThat(filterPath.matches("0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("t", nextFilters), is(true)); + assertThat(filterPath.matches("t", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); input = "w\\.0\\.0\\.t"; @@ -131,7 +131,7 @@ public void testFilterPathWithEscapedDots() { nextFilters = new ArrayList<>(); filterPath = filterPaths[0]; - assertTrue(filterPath.matches("w.0.0.t", nextFilters)); + assertTrue(filterPath.matches("w.0.0.t", nextFilters, false)); assertEquals(nextFilters.size(), 0); input = "w\\.0.0\\.t"; @@ -143,13 +143,13 @@ public void testFilterPathWithEscapedDots() { filterPath = filterPaths[0]; nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("w.0", nextFilters), is(false)); + assertThat(filterPath.matches("w.0", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("0.t", nextFilters), is(true)); + assertThat(filterPath.matches("0.t", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -161,7 +161,7 @@ public void testSimpleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(true)); + assertThat(filterPath.matches("foo", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -175,19 +175,19 @@ public void testWildcardInNameFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); - assertThat(filterPath.matches("flo", nextFilters), is(false)); + assertThat(filterPath.matches("flo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 2); - assertThat(filterPath.matches("foooo", nextFilters), is(false)); + assertThat(filterPath.matches("foooo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 3); - assertThat(filterPath.matches("boo", nextFilters), is(false)); + assertThat(filterPath.matches("boo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 3); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -199,7 +199,7 @@ public void testDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(true)); + assertThat(filterPath.matches("foo", nextFilters, false), is(true)); assertThat(filterPath.hasDoubleWildcard(), is(true)); assertEquals(nextFilters.size(), 0); } @@ -214,13 +214,13 @@ public void testStartsWithDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -234,19 +234,19 @@ public void testContainsDoubleWildcardFilterPath() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(true)); + assertThat(filterPath.matches("bar", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -261,38 +261,38 @@ public void testMultipleFilterPaths() { FilterPath filterPath = filterPaths[0]; List nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("foo", nextFilters), is(false)); + assertThat(filterPath.matches("foo", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("bar", nextFilters), is(false)); + assertThat(filterPath.matches("bar", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 2); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test2", nextFilters), is(true)); + assertThat(filterPath.matches("test2", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); // test.dot\.ted filterPath = filterPaths[0]; nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("test", nextFilters), is(false)); + assertThat(filterPath.matches("test", nextFilters, false), is(false)); assertEquals(nextFilters.size(), 1); filterPath = nextFilters.get(0); nextFilters = new ArrayList<>(); assertNotNull(filterPath); - assertThat(filterPath.matches("dot.ted", nextFilters), is(true)); + assertThat(filterPath.matches("dot.ted", nextFilters, false), is(true)); assertEquals(nextFilters.size(), 0); } @@ -335,7 +335,36 @@ public void testNoMatchesFilter() { List nextFilters = new ArrayList<>(); FilterPath filterPath = filterPaths[0]; - assertFalse(filterPath.matches(randomAlphaOfLength(10), nextFilters)); + assertFalse(filterPath.matches(randomAlphaOfLength(10), nextFilters, false)); + assertEquals(nextFilters.size(), 0); + } + + public void testDotInFieldName() { + // FilterPath match + FilterPath[] filterPaths = FilterPath.compile(singleton("foo")); + List nextFilters = new ArrayList<>(); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath not match + filterPaths = FilterPath.compile(singleton("bar")); + assertFalse(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath equals to fieldName + filterPaths = FilterPath.compile(singleton("foo.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // FilterPath longer than fieldName + filterPaths = FilterPath.compile(singleton("foo.bar.test")); + assertFalse(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + + // partial match + filterPaths = FilterPath.compile(singleton("foo.bar.test")); + assertFalse(filterPaths[0].matches("foo.bar.text", nextFilters, true)); assertEquals(nextFilters.size(), 0); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index cb9489b7745c1..6cc399f598155 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -404,7 +404,8 @@ class DataStream implements IndexAbstraction { public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering( Set.of("@timestamp"), - null + null, + false ); private final org.elasticsearch.cluster.metadata.DataStream dataStream; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 544933c29fac9..8bf20281844f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -8,6 +8,8 @@ package org.elasticsearch.cluster.routing; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -23,7 +25,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.IntConsumer; @@ -214,7 +216,7 @@ private static class ExtractFromSource extends IndexRouting { if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(metadata.getRoutingPaths()), null); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(metadata.getRoutingPaths()), null, true); } @Override @@ -224,66 +226,78 @@ public int indexShard(String id, @Nullable String routing, XContentType sourceTy } assert Transports.assertNotTransportThread("parsing the _source can get slow"); + List hashes = new ArrayList<>(); try { try (XContentParser parser = sourceType.xContent().createParser(parserConfig, source.streamInput())) { parser.nextToken(); // Move to first token if (parser.currentToken() == null) { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } - int hash = extractObject(parser); + parser.nextToken(); + extractObject(hashes, null, parser); ensureExpectedToken(null, parser.nextToken(), parser); - return hashToShardId(hash); } } catch (IOException | ParsingException e) { throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); } + return hashToShardId(hashesToHash(hashes)); } - private static int extractObject(XContentParser source) throws IOException { - ensureExpectedToken(Token.FIELD_NAME, source.nextToken(), source); - String firstFieldName = source.currentName(); - source.nextToken(); - int firstHash = extractItem(source); - if (source.currentToken() == Token.END_OBJECT) { - // Just one routing key in this object - // Use ^ like Map.Entry's hashcode - return Murmur3HashFunction.hash(firstFieldName) ^ firstHash; - } - List hashes = new ArrayList<>(); - hashes.add(new NameAndHash(firstFieldName, firstHash)); - do { + private static void extractObject(List hashes, @Nullable String path, XContentParser source) throws IOException { + while (source.currentToken() != Token.END_OBJECT) { ensureExpectedToken(Token.FIELD_NAME, source.currentToken(), source); String fieldName = source.currentName(); + String subPath = path == null ? fieldName : path + "." + fieldName; source.nextToken(); - hashes.add(new NameAndHash(fieldName, extractItem(source))); - } while (source.currentToken() != Token.END_OBJECT); - Collections.sort(hashes, Comparator.comparing(nameAndHash -> nameAndHash.name)); - /* - * This is the same as Arrays.hash(Map.Entry) but we're - * writing it out so for extra paranoia. Changing this will change how - * documents are routed and we don't want a jdk update that modifies Arrays - * or Map.Entry to sneak up on us. - */ - int hash = 0; - for (NameAndHash nameAndHash : hashes) { - int thisHash = Murmur3HashFunction.hash(nameAndHash.name) ^ nameAndHash.hash; - hash = 31 * hash + thisHash; + extractItem(hashes, subPath, source); } - return hash; } - private static int extractItem(XContentParser source) throws IOException { - if (source.currentToken() == Token.START_OBJECT) { - int hash = extractObject(source); - source.nextToken(); - return hash; + private static void extractItem(List hashes, String path, XContentParser source) throws IOException { + switch (source.currentToken()) { + case START_OBJECT: + source.nextToken(); + extractObject(hashes, path, source); + source.nextToken(); + break; + case VALUE_STRING: + hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); + source.nextToken(); + break; + case VALUE_NULL: + source.nextToken(); + break; + default: + throw new ParsingException( + source.getTokenLocation(), + "Routing values must be strings but found [{}]", + source.currentToken() + ); } - if (source.currentToken() == Token.VALUE_STRING) { - int hash = Murmur3HashFunction.hash(source.text()); - source.nextToken(); - return hash; + } + + private static int hash(BytesRef ref) { + return StringHelper.murmurhash3_x86_32(ref, 0); + } + + private static int hashesToHash(List hashes) { + Collections.sort(hashes); + Iterator itr = hashes.iterator(); + if (itr.hasNext() == false) { + throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); + } + NameAndHash prev = itr.next(); + int hash = hash(prev.name) ^ prev.hash; + while (itr.hasNext()) { + NameAndHash next = itr.next(); + if (prev.name.equals(next.name)) { + throw new IllegalArgumentException("Duplicate routing dimension for [" + next.name + "]"); + } + int thisHash = hash(next.name) ^ next.hash; + hash = 31 * hash + thisHash; + prev = next; } - throw new ParsingException(source.getTokenLocation(), "Routing values must be strings but found [{}]", source.currentToken()); + return hash; } @Override @@ -316,5 +330,10 @@ private String error(String operation) { } } - private record NameAndHash(String name, int hash) {} + private static record NameAndHash(BytesRef name, int hash) implements Comparable { + @Override + public int compareTo(NameAndHash o) { + return name.compareTo(o.name); + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 0c02f9770b72f..80a19f4dc7994 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -232,7 +232,7 @@ public static Map convertToMap( @Nullable Set include, @Nullable Set exclude ) throws ElasticsearchParseException { - try (XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude), input)) { + try (XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input)) { return ordered ? parser.mapOrdered() : parser.map(); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); @@ -266,7 +266,7 @@ public static Map convertToMap( ) throws ElasticsearchParseException { try ( XContentParser parser = xContent.createParser( - XContentParserConfiguration.EMPTY.withFiltering(include, exclude), + XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), bytes, offset, length diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index b18c90fecc326..5509ee5cc0d4d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.cluster.routing; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.Version; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -513,7 +515,7 @@ public void testRoutingPathOneSub() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), - Math.floorMod(hash(List.of("foo", List.of("bar", "cat"))), shards) + Math.floorMod(hash(List.of("foo.bar", "cat")), shards) ); } @@ -523,10 +525,16 @@ public void testRoutingPathManySubs() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("a", "cat"), "bar", Map.of("thing", "yay", "this", "too")), - Math.floorMod(hash(List.of("bar", List.of("thing", "yay", "this", "too"), "foo", List.of("a", "cat"))), shards) + Math.floorMod(hash(List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat")), shards) ); } + public void testRoutingPathDotInName() throws IOException { + int shards = between(2, 1000); + IndexRouting routing = indexRoutingForPath(shards, "foo.bar"); + assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), Math.floorMod(hash(List.of("foo.bar", "cat")), shards)); + } + public void testRoutingPathBwc() throws IOException { Version version = VersionUtils.randomIndexCompatibleVersion(random()); IndexRouting routing = indexRoutingForPath(version, 8, "dim.*,other.*,top"); @@ -538,12 +546,13 @@ public void testRoutingPathBwc() throws IOException { * versions of Elasticsearch must continue to route based on the * version on the index. */ - assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 0); + assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 4); assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 5); assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 4); - assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 5); - assertIndexShard(routing, Map.of("top", "a"), 3); - assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 2); + assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 7); + assertIndexShard(routing, Map.of("top", "a"), 5); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 0); + assertIndexShard(routing, Map.of("dim.a", "a"), 4); } private IndexRouting indexRoutingForPath(int shards, String path) { @@ -560,8 +569,8 @@ private IndexRouting indexRoutingForPath(Version createdVersion, int shards, Str ); } - private void assertIndexShard(IndexRouting routing, Map source, int id) throws IOException { - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(source)), equalTo(id)); + private void assertIndexShard(IndexRouting routing, Map source, int expected) throws IOException { + assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(source)), equalTo(expected)); } private BytesReference source(Map doc) throws IOException { @@ -581,24 +590,14 @@ private BytesReference source(Map doc) throws IOException { /** * Build the hash we expect from the extracter. */ - private int hash(List keysAndValues) { + private int hash(List keysAndValues) { assertThat(keysAndValues.size() % 2, equalTo(0)); int hash = 0; for (int i = 0; i < keysAndValues.size(); i += 2) { - int thisHash = Murmur3HashFunction.hash(keysAndValues.get(i).toString()) ^ expectedValueHash(keysAndValues.get(i + 1)); - hash = hash * 31 + thisHash; + int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i)), 0); + int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1)), 0); + hash = hash * 31 + (keyHash ^ valueHash); } return hash; } - - private int expectedValueHash(Object value) { - if (value instanceof List) { - return hash((List) value); - } - if (value instanceof String) { - return Murmur3HashFunction.hash((String) value); - } - throw new IllegalArgumentException("Unsupported value: " + value); - } - } From cff63d6ac3b4a746cd2a89ee739b3b25b7e72429 Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Thu, 27 Jan 2022 16:55:25 +0800 Subject: [PATCH 2/8] add wildcard and double wildcard tests --- .../AbstractXContentFilteringTestCase.java | 3 +- .../support/filtering/FilterPathTests.java | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 9744cf533010e..55980f258c5a4 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -156,8 +156,7 @@ public void testDotInExcludedFieldNamePatternConfigured() throws IOException { builder -> builder.startObject().endObject(), builder -> builder.startObject().field("foo.bar", "test").endObject(), emptySet(), - singleton("*.bar"), -// singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), + singleton(randomFrom("foo.*", "*.bar", "f*.bar", "foo.*ar", "*.*")), true ); } diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java index 716bbce5bdfb0..465bab256ac3b 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/FilterPathTests.java @@ -366,5 +366,40 @@ public void testDotInFieldName() { filterPaths = FilterPath.compile(singleton("foo.bar.test")); assertFalse(filterPaths[0].matches("foo.bar.text", nextFilters, true)); assertEquals(nextFilters.size(), 0); + + // wildcard + filterPaths = FilterPath.compile(singleton("*.bar")); + assertFalse(filterPaths[0].matches("foo", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + filterPaths = FilterPath.compile(singleton("*.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("f*.bar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("foo.*")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("foo.*ar")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + filterPaths = FilterPath.compile(singleton("*.*")); + assertTrue(filterPaths[0].matches("foo.bar", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + + // test double wildcard + filterPaths = FilterPath.compile(singleton("**.c")); + assertFalse(filterPaths[0].matches("a.b", nextFilters, true)); + assertEquals(nextFilters.size(), 1); + nextFilters.clear(); + assertTrue(filterPaths[0].matches("a.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.d.c", nextFilters, true)); + assertEquals(nextFilters.size(), 0); + assertTrue(filterPaths[0].matches("a.b.c.d", nextFilters, true)); + assertEquals(nextFilters.size(), 0); } } From a60340d0c9b3f68fa66625a0bdc1a71b216a2cc6 Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Thu, 27 Jan 2022 17:09:45 +0800 Subject: [PATCH 3/8] fix checkstyle --- .../org/elasticsearch/common/xcontent/XContentHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 80a19f4dc7994..d120983ce562c 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -232,7 +232,9 @@ public static Map convertToMap( @Nullable Set include, @Nullable Set exclude ) throws ElasticsearchParseException { - try (XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input)) { + try ( + XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input) + ) { return ordered ? parser.mapOrdered() : parser.map(); } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse content to map", e); From e90d68a45ce40504d78e7a2c8512071eef04ccf0 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 27 Jan 2022 12:14:20 -0500 Subject: [PATCH 4/8] Another test --- .../AbstractXContentFilteringTestCase.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 55980f258c5a4..5d31160ba5d42 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.util.Arrays; import java.util.Set; +import java.util.stream.IntStream; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; +import static java.util.stream.Collectors.joining; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; @@ -221,6 +223,37 @@ public void testDotInStarMatchDotsInNamesConfigured() throws IOException { ); } + public void testTwoDotsInIncludedFieldNameUnconfigured() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar.baz"), + emptySet(), + false + ); + } + + public void testTwoDotsInIncludedFieldNameConfigured() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar.baz"), + emptySet(), + true + ); + } + + public void testManyDotsInIncludedFieldName() throws IOException { + String name = IntStream.rangeClosed(1, 40000).mapToObj(i -> "a").collect(joining(".")); + testFilter( + builder -> builder.startObject().field(name, "test").endObject(), + builder -> builder.startObject().field(name, "test").endObject(), + singleton(name), + emptySet(), + true + ); + } + @Override protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { testFilter(expected, sample, includes, excludes, false); From 6dacf9c95ffcc3283339a6c4f601fd3a9680a792 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 27 Jan 2022 12:22:17 -0500 Subject: [PATCH 5/8] More --- .../AbstractXContentFilteringTestCase.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 5d31160ba5d42..beedf64d14e32 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -254,6 +254,46 @@ public void testManyDotsInIncludedFieldName() throws IOException { ); } + public void testDotsInIncludedFieldNamePrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("foo.bar"), + emptySet(), + true + ); + } + + public void testDotsInExcludedFieldNamePrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("foo.bar"), + true + ); + } + + public void testDotsInIncludedFieldNamePatternPrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("f*.*r"), + emptySet(), + true + ); + } + + public void testDotsInExcludedFieldNamePatternPrefixMatch() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("f*.*r"), + true + ); + } + @Override protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { testFilter(expected, sample, includes, excludes, false); From 90758bf4e02bf4b2db8e4f2937e5f9cf31acf461 Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Fri, 28 Jan 2022 13:53:55 +0800 Subject: [PATCH 6/8] rollback some changes that does not belong to dot-matching --- .../support/filtering/FilterPath.java | 4 +- .../cluster/routing/IndexRouting.java | 101 +++++++----------- .../cluster/routing/IndexRoutingTests.java | 43 ++++---- 3 files changed, 64 insertions(+), 84 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java index 658ad1eb5ab32..350611ef01a85 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/support/filtering/FilterPath.java @@ -77,7 +77,6 @@ boolean matches(String name, List nextFilters, boolean matchFieldNam } // match dot first - FilterPath termNode; if (matchFieldNamesWithDots) { // contains dot and not the first or last char int dotIndex = name.indexOf('.'); @@ -85,8 +84,7 @@ boolean matches(String name, List nextFilters, boolean matchFieldNam return matchFieldNamesWithDots(name, dotIndex, nextFilters); } } - - termNode = termsChildren.get(name); + FilterPath termNode = termsChildren.get(name); if (termNode != null) { if (termNode.isFinalNode()) { return true; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 8bf20281844f6..64a2a2b093468 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -8,8 +8,6 @@ package org.elasticsearch.cluster.routing; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.StringHelper; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -25,7 +23,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; +import java.util.Comparator; import java.util.List; import java.util.Set; import java.util.function.IntConsumer; @@ -226,78 +224,66 @@ public int indexShard(String id, @Nullable String routing, XContentType sourceTy } assert Transports.assertNotTransportThread("parsing the _source can get slow"); - List hashes = new ArrayList<>(); try { try (XContentParser parser = sourceType.xContent().createParser(parserConfig, source.streamInput())) { parser.nextToken(); // Move to first token if (parser.currentToken() == null) { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } - parser.nextToken(); - extractObject(hashes, null, parser); + int hash = extractObject(parser); ensureExpectedToken(null, parser.nextToken(), parser); + return hashToShardId(hash); } } catch (IOException | ParsingException e) { throw new IllegalArgumentException("Error extracting routing: " + e.getMessage(), e); } - return hashToShardId(hashesToHash(hashes)); } - private static void extractObject(List hashes, @Nullable String path, XContentParser source) throws IOException { - while (source.currentToken() != Token.END_OBJECT) { + private static int extractObject(XContentParser source) throws IOException { + ensureExpectedToken(Token.FIELD_NAME, source.nextToken(), source); + String firstFieldName = source.currentName(); + source.nextToken(); + int firstHash = extractItem(source); + if (source.currentToken() == Token.END_OBJECT) { + // Just one routing key in this object + // Use ^ like Map.Entry's hashcode + return Murmur3HashFunction.hash(firstFieldName) ^ firstHash; + } + List hashes = new ArrayList<>(); + hashes.add(new NameAndHash(firstFieldName, firstHash)); + do { ensureExpectedToken(Token.FIELD_NAME, source.currentToken(), source); String fieldName = source.currentName(); - String subPath = path == null ? fieldName : path + "." + fieldName; source.nextToken(); - extractItem(hashes, subPath, source); - } - } - - private static void extractItem(List hashes, String path, XContentParser source) throws IOException { - switch (source.currentToken()) { - case START_OBJECT: - source.nextToken(); - extractObject(hashes, path, source); - source.nextToken(); - break; - case VALUE_STRING: - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); - source.nextToken(); - break; - case VALUE_NULL: - source.nextToken(); - break; - default: - throw new ParsingException( - source.getTokenLocation(), - "Routing values must be strings but found [{}]", - source.currentToken() - ); + hashes.add(new NameAndHash(fieldName, extractItem(source))); + } while (source.currentToken() != Token.END_OBJECT); + Collections.sort(hashes, Comparator.comparing(nameAndHash -> nameAndHash.name)); + /* + * This is the same as Arrays.hash(Map.Entry) but we're + * writing it out so for extra paranoia. Changing this will change how + * documents are routed and we don't want a jdk update that modifies Arrays + * or Map.Entry to sneak up on us. + */ + int hash = 0; + for (NameAndHash nameAndHash : hashes) { + int thisHash = Murmur3HashFunction.hash(nameAndHash.name) ^ nameAndHash.hash; + hash = 31 * hash + thisHash; } + return hash; } - private static int hash(BytesRef ref) { - return StringHelper.murmurhash3_x86_32(ref, 0); - } - - private static int hashesToHash(List hashes) { - Collections.sort(hashes); - Iterator itr = hashes.iterator(); - if (itr.hasNext() == false) { - throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); + private static int extractItem(XContentParser source) throws IOException { + if (source.currentToken() == Token.START_OBJECT) { + int hash = extractObject(source); + source.nextToken(); + return hash; } - NameAndHash prev = itr.next(); - int hash = hash(prev.name) ^ prev.hash; - while (itr.hasNext()) { - NameAndHash next = itr.next(); - if (prev.name.equals(next.name)) { - throw new IllegalArgumentException("Duplicate routing dimension for [" + next.name + "]"); - } - int thisHash = hash(next.name) ^ next.hash; - hash = 31 * hash + thisHash; - prev = next; + if (source.currentToken() == Token.VALUE_STRING) { + int hash = Murmur3HashFunction.hash(source.text()); + source.nextToken(); + return hash; } - return hash; + throw new ParsingException(source.getTokenLocation(), "Routing values must be strings but found [{}]", source.currentToken()); } @Override @@ -330,10 +316,5 @@ private String error(String operation) { } } - private static record NameAndHash(BytesRef name, int hash) implements Comparable { - @Override - public int compareTo(NameAndHash o) { - return name.compareTo(o.name); - } - } + private record NameAndHash(String name, int hash) {} } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 5509ee5cc0d4d..b18c90fecc326 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -7,8 +7,6 @@ */ package org.elasticsearch.cluster.routing; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.StringHelper; import org.elasticsearch.Version; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -515,7 +513,7 @@ public void testRoutingPathOneSub() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("bar", "cat"), "baz", "dog"), - Math.floorMod(hash(List.of("foo.bar", "cat")), shards) + Math.floorMod(hash(List.of("foo", List.of("bar", "cat"))), shards) ); } @@ -525,16 +523,10 @@ public void testRoutingPathManySubs() throws IOException { assertIndexShard( routing, Map.of("foo", Map.of("a", "cat"), "bar", Map.of("thing", "yay", "this", "too")), - Math.floorMod(hash(List.of("bar.thing", "yay", "bar.this", "too", "foo.a", "cat")), shards) + Math.floorMod(hash(List.of("bar", List.of("thing", "yay", "this", "too"), "foo", List.of("a", "cat"))), shards) ); } - public void testRoutingPathDotInName() throws IOException { - int shards = between(2, 1000); - IndexRouting routing = indexRoutingForPath(shards, "foo.bar"); - assertIndexShard(routing, Map.of("foo.bar", "cat", "baz", "dog"), Math.floorMod(hash(List.of("foo.bar", "cat")), shards)); - } - public void testRoutingPathBwc() throws IOException { Version version = VersionUtils.randomIndexCompatibleVersion(random()); IndexRouting routing = indexRoutingForPath(version, 8, "dim.*,other.*,top"); @@ -546,13 +538,12 @@ public void testRoutingPathBwc() throws IOException { * versions of Elasticsearch must continue to route based on the * version on the index. */ - assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 4); + assertIndexShard(routing, Map.of("dim", Map.of("a", "a")), 0); assertIndexShard(routing, Map.of("dim", Map.of("a", "b")), 5); assertIndexShard(routing, Map.of("dim", Map.of("c", "d")), 4); - assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 7); - assertIndexShard(routing, Map.of("top", "a"), 5); - assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 0); - assertIndexShard(routing, Map.of("dim.a", "a"), 4); + assertIndexShard(routing, Map.of("other", Map.of("a", "a")), 5); + assertIndexShard(routing, Map.of("top", "a"), 3); + assertIndexShard(routing, Map.of("dim", Map.of("c", "d"), "top", "b"), 2); } private IndexRouting indexRoutingForPath(int shards, String path) { @@ -569,8 +560,8 @@ private IndexRouting indexRoutingForPath(Version createdVersion, int shards, Str ); } - private void assertIndexShard(IndexRouting routing, Map source, int expected) throws IOException { - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(source)), equalTo(expected)); + private void assertIndexShard(IndexRouting routing, Map source, int id) throws IOException { + assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(source)), equalTo(id)); } private BytesReference source(Map doc) throws IOException { @@ -590,14 +581,24 @@ private BytesReference source(Map doc) throws IOException { /** * Build the hash we expect from the extracter. */ - private int hash(List keysAndValues) { + private int hash(List keysAndValues) { assertThat(keysAndValues.size() % 2, equalTo(0)); int hash = 0; for (int i = 0; i < keysAndValues.size(); i += 2) { - int keyHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i)), 0); - int valueHash = StringHelper.murmurhash3_x86_32(new BytesRef(keysAndValues.get(i + 1)), 0); - hash = hash * 31 + (keyHash ^ valueHash); + int thisHash = Murmur3HashFunction.hash(keysAndValues.get(i).toString()) ^ expectedValueHash(keysAndValues.get(i + 1)); + hash = hash * 31 + thisHash; } return hash; } + + private int expectedValueHash(Object value) { + if (value instanceof List) { + return hash((List) value); + } + if (value instanceof String) { + return Murmur3HashFunction.hash((String) value); + } + throw new IllegalArgumentException("Unsupported value: " + value); + } + } From 1a1fd491f87a96bde4010733af9d743d96d30cec Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Fri, 28 Jan 2022 18:17:10 +0800 Subject: [PATCH 7/8] AbstractXContentFilteringTestCase add double wildcard tests for dots in includes and excludes --- .../AbstractXContentFilteringTestCase.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index beedf64d14e32..52c0f16bbca43 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -294,6 +294,35 @@ public void testDotsInExcludedFieldNamePatternPrefixMatch() throws IOException { ); } + public void testDotsAndDoubleWildcardInIncludedFieldName() throws IOException { + testFilter( + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + singleton("**.baz"), + emptySet(), + true + ); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/80160") + public void testDotsAndDoubleWildcardInExcludedFieldName() throws IOException { + testFilter( + builder -> builder.startObject().endObject(), + builder -> builder.startObject().field("foo.bar.baz", "test").endObject(), + emptySet(), + singleton("**.baz"), + true + ); + // bug of double wildcard in excludes report in https://github.com/FasterXML/jackson-core/issues/700 + testFilter( + builder -> builder.startObject().startObject("foo").field("baz", "test").endObject().endObject(), + builder -> builder.startObject().startObject("foo").field("bar", "test").field("baz", "test").endObject().endObject(), + emptySet(), + singleton("**.bar"), + true + ); + } + @Override protected final void testFilter(Builder expected, Builder sample, Set includes, Set excludes) throws IOException { testFilter(expected, sample, includes, excludes, false); From 3b8d12753824a36063d27f8968d4050f1c53a8cd Mon Sep 17 00:00:00 2001 From: mushaoqiong Date: Fri, 28 Jan 2022 23:44:29 +0800 Subject: [PATCH 8/8] Make dot number samller to avoid StackOverFolwErr for now --- .../support/filtering/AbstractXContentFilteringTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 52c0f16bbca43..5d535b2446358 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -244,7 +244,7 @@ public void testTwoDotsInIncludedFieldNameConfigured() throws IOException { } public void testManyDotsInIncludedFieldName() throws IOException { - String name = IntStream.rangeClosed(1, 40000).mapToObj(i -> "a").collect(joining(".")); + String name = IntStream.rangeClosed(1, 100).mapToObj(i -> "a").collect(joining(".")); testFilter( builder -> builder.startObject().field(name, "test").endObject(), builder -> builder.startObject().field(name, "test").endObject(),