From 1869db8dfab7ded219d6045c9a37e0cd79b73aa8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 15 Mar 2017 17:39:50 -0400 Subject: [PATCH 01/28] Move asciifolding into analysis-common --- .../indices/analysis/AnalysisModule.java | 2 - .../index/analysis/AnalysisFactoryTests.java | 1 + modules/analysis-common/build.gradle | 23 ++++++++++ .../ASCIIFoldingTokenFilterFactory.java | 5 +- .../analysis/common/CommonAnalysisPlugin.java | 37 +++++++++++++++ .../CommonAnalysisClientYamlTestSuiteIT.java | 38 +++++++++++++++ .../common/CommonAnalysisFactoryTests.java | 46 +++++++++++++++++++ .../test/analysis-common/10_basic.yaml | 11 +++++ .../analysis/AnalysisPolishFactoryTests.java | 9 ++-- settings.gradle | 5 +- .../AnalysisFactoryTestCase.java | 4 +- 11 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 modules/analysis-common/build.gradle rename {core/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/ASCIIFoldingTokenFilterFactory.java (91%) create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yaml diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index 61950942e6076..c1883554ea0e5 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.ApostropheFilterFactory; @@ -205,7 +204,6 @@ private NamedRegistry> setupTokenFilters(Li NamedRegistry> tokenFilters = new NamedRegistry<>("token_filter"); tokenFilters.register("stop", StopTokenFilterFactory::new); tokenFilters.register("reverse", ReverseTokenFilterFactory::new); - tokenFilters.register("asciifolding", ASCIIFoldingTokenFilterFactory::new); tokenFilters.register("length", LengthTokenFilterFactory::new); tokenFilters.register("lowercase", LowerCaseTokenFilterFactory::new); tokenFilters.register("uppercase", UpperCaseTokenFilterFactory::new); diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java index 6893fda75b84e..8c2f0ee1d51da 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java @@ -23,4 +23,5 @@ public class AnalysisFactoryTests extends AnalysisFactoryTestCase { // tests are inherited + // NOCOMMIT this should be dropped from core when there are no analyzers exposed there } diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle new file mode 100644 index 0000000000000..391b74934c97d --- /dev/null +++ b/modules/analysis-common/build.gradle @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Adds "built in" analyzers to Elasticsearch.' + classname 'org.elasticsearch.analysis.common.CommonAnalysisPlugin' +} diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java similarity index 91% rename from core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java index 5e53a86129a97..3991c7f44bd1f 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; @@ -25,6 +25,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.TokenFilterFactory; /** * Factory for ASCIIFoldingFilter. diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java new file mode 100644 index 0000000000000..533791578cad7 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.HashMap; +import java.util.Map; + +public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { + @Override + public Map> getTokenFilters() { + Map> filters = new HashMap<>(); + filters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new); + return filters; + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..c77966e81b0f5 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.analysis.common; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.io.IOException; + +public class CommonAnalysisClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + public CommonAnalysisClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java new file mode 100644 index 0000000000000..6793278a42d51 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.AnalysisFactoryTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase { + @Override + protected Map> getTokenizers() { + Map> tokenizers = new HashMap<>(super.getTokenizers()); + return tokenizers; + } + + @Override + protected Map> getTokenFilters() { + Map> filters = new HashMap<>(super.getTokenFilters()); + filters.put("asciifolding", ASCIIFoldingTokenFilterFactory.class); + return filters; + } + + @Override + protected Map> getCharFilters() { + Map> filters = new HashMap<>(super.getCharFilters()); + return filters; + } +} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yaml new file mode 100644 index 0000000000000..d27a0861b2e38 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yaml @@ -0,0 +1,11 @@ +"Module loaded": + - do: + cluster.state: {} + + # Get master node id + - set: { master_node: master } + + - do: + nodes.info: {} + + - match: { nodes.$master.modules.0.name: analysis-common } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java index e68cb260b0bf0..8301529627670 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java @@ -19,14 +19,9 @@ package org.elasticsearch.index.analysis; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.Tokenizer; import org.elasticsearch.AnalysisFactoryTestCase; import org.elasticsearch.Version; @@ -37,6 +32,10 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + public class AnalysisPolishFactoryTests extends AnalysisFactoryTestCase { @Override diff --git a/settings.gradle b/settings.gradle index c0996f86a04fc..6b28fe1551e19 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,14 +25,15 @@ List projects = [ 'test:fixtures:hdfs-fixture', 'test:logger-usage', 'modules:aggs-matrix-stats', + 'modules:analysis-common', 'modules:ingest-common', 'modules:lang-expression', 'modules:lang-mustache', 'modules:lang-painless', - 'modules:transport-netty4', - 'modules:reindex', 'modules:percolator', + 'modules:reindex', 'modules:repository-url', + 'modules:transport-netty4', 'plugins:analysis-icu', 'plugins:analysis-kuromoji', 'plugins:analysis-phonetic', diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 76d33a6c42565..4601a30f94165 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -26,8 +26,8 @@ import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.elasticsearch.Version; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory; import org.elasticsearch.index.analysis.ApostropheFilterFactory; import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory; import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory; @@ -178,7 +178,7 @@ private static String toCamelCase(String s) { .put("apostrophe", ApostropheFilterFactory.class) .put("arabicnormalization", ArabicNormalizationFilterFactory.class) .put("arabicstem", ArabicStemTokenFilterFactory.class) - .put("asciifolding", ASCIIFoldingTokenFilterFactory.class) + .put("asciifolding", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. .put("brazilianstem", BrazilianStemTokenFilterFactory.class) .put("bulgarianstem", StemmerTokenFilterFactory.class) .put("cjkbigram", CJKBigramFilterFactory.class) From a4ed494112b93aab315761adb984e2db4089e97e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 15 Mar 2017 18:35:53 -0400 Subject: [PATCH 02/28] Tests --- .../common}/ASCIIFoldingTokenFilterFactoryTests.java | 9 ++++++--- .../BaseWordDelimiterTokenFilterFactoryTestCase.java | 6 +++++- .../WordDelimiterGraphTokenFilterFactoryTests.java | 4 +++- .../common}/WordDelimiterTokenFilterFactoryTests.java | 4 +++- .../index/analysis/AnalysisTestsHelper.java | 8 ++++---- 5 files changed, 21 insertions(+), 10 deletions(-) rename {core/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/ASCIIFoldingTokenFilterFactoryTests.java (90%) rename {core/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/BaseWordDelimiterTokenFilterFactoryTestCase.java (96%) rename {core/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/WordDelimiterGraphTokenFilterFactoryTests.java (96%) rename {core/src/test/java/org/elasticsearch/index/analysis => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/WordDelimiterTokenFilterFactoryTests.java (93%) rename {core/src/test => test/framework/src/main}/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java (91%) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java similarity index 90% rename from core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java index 973225df180f3..813311c0857b9 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java @@ -17,12 +17,15 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.analysis.AnalysisTestsHelper; +import org.elasticsearch.index.analysis.MultiTermAwareComponent; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTokenStreamTestCase; @@ -34,7 +37,7 @@ public void testDefault() throws IOException { ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding"); String source = "Ansprüche"; String[] expected = new String[]{"Anspruche"}; @@ -48,7 +51,7 @@ public void testPreserveOriginal() throws IOException { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") .put("index.analysis.filter.my_ascii_folding.preserve_original", true) - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding"); String source = "Ansprüche"; String[] expected = new String[]{"Anspruche", "Ansprüche"}; diff --git a/core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java similarity index 96% rename from core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index 713e9424759a0..a6277dbd0f590 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -16,13 +16,17 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.analysis.AnalysisTestsHelper; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.analysis.WordDelimiterGraphTokenFilterFactory; +import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTokenStreamTestCase; diff --git a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java similarity index 96% rename from core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index 2ae4267104a05..a4e04aadba342 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -16,13 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.analysis.AnalysisTestsHelper; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import java.io.IOException; diff --git a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java similarity index 93% rename from core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java index 1e919e00bbb29..6c6d529fc28ec 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java @@ -16,13 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.analysis.AnalysisTestsHelper; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.test.ESTestCase; import java.io.IOException; diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java similarity index 91% rename from core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java rename to test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java index a60c21c1a7e6d..d55ea8a39d9b0 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java @@ -25,13 +25,13 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; - -import static java.util.Collections.emptyList; +import java.util.Arrays; public class AnalysisTestsHelper { @@ -45,12 +45,12 @@ public static ESTestCase.TestAnalysis createTestAnalysisFromClassPath(Path baseD } public static ESTestCase.TestAnalysis createTestAnalysisFromSettings( - Settings settings) throws IOException { + Settings settings, AnalysisPlugin... plugins) throws IOException { if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) { settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); } IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisRegistry analysisRegistry = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry(); + AnalysisRegistry analysisRegistry = new AnalysisModule(new Environment(settings), Arrays.asList(plugins)).getAnalysisRegistry(); return new ESTestCase.TestAnalysis(analysisRegistry.build(indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), From 1e4a2d91b4d5a8fd367428780bcccc71bbb2e127 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 16 Mar 2017 10:16:35 -0400 Subject: [PATCH 03/28] Fix some tests --- .../analysis/EdgeNGramTokenFilterFactory.java | 5 ++ .../index/analysis/TokenFilterFactory.java | 4 + .../indices/analysis/AnalysisModule.java | 4 - .../highlight/FragmentBuilderHelper.java | 9 +- .../index/analysis/AnalysisRegistryTests.java | 87 ++++++++++++------- .../analysis/common/CommonAnalysisPlugin.java | 2 + .../WordDelimiterGraphTokenFilterFactory.java | 6 +- .../WordDelimiterTokenFilterFactory.java | 9 +- ...rdDelimiterTokenFilterFactoryTestCase.java | 2 - ...DelimiterGraphTokenFilterFactoryTests.java | 2 +- .../WordDelimiterTokenFilterFactoryTests.java | 2 +- .../AnalysisFactoryTestCase.java | 7 +- 12 files changed, 86 insertions(+), 53 deletions(-) rename {core/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/WordDelimiterGraphTokenFilterFactory.java (95%) rename {core/src/main/java/org/elasticsearch/index/analysis => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/WordDelimiterTokenFilterFactory.java (97%) diff --git a/core/src/main/java/org/elasticsearch/index/analysis/EdgeNGramTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/EdgeNGramTokenFilterFactory.java index 9d287d90c8393..1d3b8e296ec35 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/EdgeNGramTokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/EdgeNGramTokenFilterFactory.java @@ -71,4 +71,9 @@ public TokenStream create(TokenStream tokenStream) { return result; } + + @Override + public boolean breaksFastVectorHighlighter() { + return true; + } } \ No newline at end of file diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index 8c976646b8554..6456bd29258f7 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -26,4 +26,8 @@ public interface TokenFilterFactory { String name(); TokenStream create(TokenStream tokenStream); + + default boolean breaksFastVectorHighlighter() { + return false; + } } diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java index c1883554ea0e5..c494c4cae9c35 100644 --- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java +++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java @@ -139,8 +139,6 @@ import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory; import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider; import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory; -import org.elasticsearch.index.analysis.WordDelimiterGraphTokenFilterFactory; -import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; import org.elasticsearch.plugins.AnalysisPlugin; @@ -223,8 +221,6 @@ private NamedRegistry> setupTokenFilters(Li tokenFilters.register("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new)); tokenFilters.register("snowball", SnowballTokenFilterFactory::new); tokenFilters.register("stemmer", StemmerTokenFilterFactory::new); - tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new); - tokenFilters.register("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new); tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new); tokenFilters.register("elision", ElisionTokenFilterFactory::new); tokenFilters.register("flatten_graph", FlattenGraphTokenFilterFactory::new); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index ac0dab3a638fe..0d89e611d98ca 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -26,15 +26,9 @@ import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; import org.apache.lucene.search.vectorhighlight.FragmentsBuilder; import org.apache.lucene.util.CollectionUtil; -import org.apache.lucene.util.Version; import org.elasticsearch.index.analysis.CustomAnalyzer; -import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory; -import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory; -import org.elasticsearch.index.analysis.NGramTokenFilterFactory; -import org.elasticsearch.index.analysis.NGramTokenizerFactory; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; import org.elasticsearch.index.mapper.FieldMapper; import java.util.Comparator; @@ -91,8 +85,7 @@ private static boolean containsBrokenAnalysis(Analyzer analyzer) { final CustomAnalyzer a = (CustomAnalyzer) analyzer; TokenFilterFactory[] tokenFilters = a.tokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { - if (tokenFilterFactory instanceof WordDelimiterTokenFilterFactory - || tokenFilterFactory instanceof EdgeNGramTokenFilterFactory) { + if (tokenFilterFactory.breaksFastVectorHighlighter()) { return true; } } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 9d9631e1b006c..6d161e6b4bd59 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -21,10 +21,13 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.util.CharArraySet; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -32,6 +35,8 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.VersionUtils; @@ -45,6 +50,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -144,51 +150,74 @@ public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { "instead for index [index]"); } + /** + * Tests that {@code camelCase} filter names and {@code snake_case} filter names don't collide. + */ public void testConfigureCamelCaseTokenFilter() throws IOException { Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") - .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") - .put("index.analysis.analyzer.custom_analyzer_1.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build(); + .put("index.analysis.filter.testFilter.type", "mock") + .put("index.analysis.filter.test_filter.type", "mock") + .put("index.analysis.analyzer.custom_analyzer_with_camel_case.tokenizer", "standard") + .putArray("index.analysis.analyzer.custom_analyzer_with_camel_case.filter", "lowercase", "testFilter") + .put("index.analysis.analyzer.custom_analyzer_with_snake_case.tokenizer", "standard") + .putArray("index.analysis.analyzer.custom_analyzer_with_snake_case.filter", "lowercase", "test_filter").build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - IndexAnalyzers indexAnalyzers = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry() + /* The snake_case version of the name should not filter out any stopwords while the + * camelCase version will filter out English stopwords. */ + AnalysisPlugin plugin = new AnalysisPlugin() { + class WordDelimiterFactory extends AbstractTokenFilterFactory { + public WordDelimiterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + if (name().equals("test_filter")) { + return new MockTokenFilter(tokenStream, MockTokenFilter.EMPTY_STOPSET); + } + return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET); + } + } + @Override + public Map> getTokenFilters() { + Map> filters = new HashMap<>(); + filters.put("mock", WordDelimiterFactory::new); + return filters; + } + }; + IndexAnalyzers indexAnalyzers = new AnalysisModule(new Environment(settings), singletonList(plugin)).getAnalysisRegistry() .build(idxSettings); - try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer")) { + + // This shouldn't contain English stopwords + try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer_with_camel_case")) { assertNotNull(custom_analyser); - TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee"); + TokenStream tokenStream = custom_analyser.tokenStream("foo", "has a foo"); tokenStream.reset(); CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); - List token = new ArrayList<>(); - while(tokenStream.incrementToken()) { - token.add(charTermAttribute.toString()); - } - assertEquals(token.toString(), 2, token.size()); - assertEquals("j2se", token.get(0)); - assertEquals("j2ee", token.get(1)); + assertTrue(tokenStream.incrementToken()); + assertEquals("has", charTermAttribute.toString()); + assertTrue(tokenStream.incrementToken()); + assertEquals("foo", charTermAttribute.toString()); + assertFalse(tokenStream.incrementToken()); } - try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer_1")) { + // This *should* contain English stopwords + try (NamedAnalyzer custom_analyser = indexAnalyzers.get("custom_analyzer_with_snake_case")) { assertNotNull(custom_analyser); - TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee"); + TokenStream tokenStream = custom_analyser.tokenStream("foo", "has a foo"); tokenStream.reset(); CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); - List token = new ArrayList<>(); - while(tokenStream.incrementToken()) { - token.add(charTermAttribute.toString()); - } - assertEquals(token.toString(), 6, token.size()); - assertEquals("j", token.get(0)); - assertEquals("2", token.get(1)); - assertEquals("se", token.get(2)); - assertEquals("j", token.get(3)); - assertEquals("2", token.get(4)); - assertEquals("ee", token.get(5)); + assertTrue(tokenStream.incrementToken()); + assertEquals("has", charTermAttribute.toString()); + assertTrue(tokenStream.incrementToken()); + assertEquals("a", charTermAttribute.toString()); + assertTrue(tokenStream.incrementToken()); + assertEquals("foo", charTermAttribute.toString()); + assertFalse(tokenStream.incrementToken()); } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 533791578cad7..bfd1bbdcc97b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -32,6 +32,8 @@ public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { public Map> getTokenFilters() { Map> filters = new HashMap<>(); filters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new); + filters.put("word_delimiter", WordDelimiterTokenFilterFactory::new); + filters.put("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new); return filters; } } diff --git a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java similarity index 95% rename from core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java index 7cdc215f1b34a..0d4a5a2eed1a2 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.TokenStream; @@ -26,6 +26,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.Analysis; import java.util.List; import java.util.Set; @@ -39,7 +41,7 @@ import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_CASE_CHANGE; import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_NUMERICS; import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE; -import static org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory.parseTypes; +import static org.elasticsearch.analysis.common.WordDelimiterTokenFilterFactory.parseTypes; public class WordDelimiterGraphTokenFilterFactory extends AbstractTokenFilterFactory { diff --git a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java similarity index 97% rename from core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java index 09882072ee677..3bf3cf384bc70 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.index.analysis; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.TokenStream; @@ -26,6 +26,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.Analysis; import java.util.Collection; import java.util.List; @@ -196,4 +198,9 @@ private static String parseString(String s) { } return new String(out, 0, writePos); } + + @Override + public boolean breaksFastVectorHighlighter() { + return true; + } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index a6277dbd0f590..c6a371fca9871 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -25,8 +25,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.AnalysisTestsHelper; import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.analysis.WordDelimiterGraphTokenFilterFactory; -import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTokenStreamTestCase; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index a4e04aadba342..a640553abd44f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -41,7 +41,7 @@ public void testMultiTerms() throws IOException { .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java index 6c6d529fc28ec..44efe4e826ef5 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java @@ -42,7 +42,7 @@ public void testPartsAndCatenate() throws IOException { .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; String[] expected = new String[]{"Power", "PowerShot", "Shot" }; diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 4601a30f94165..2f723499bac06 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -20,13 +20,11 @@ package org.elasticsearch; import org.apache.lucene.analysis.en.PorterStemFilterFactory; -import org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory; import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory; import org.apache.lucene.analysis.snowball.SnowballPorterFilterFactory; import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; -import org.elasticsearch.Version; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.index.analysis.ApostropheFilterFactory; import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory; @@ -92,7 +90,6 @@ import org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory; import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory; import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory; -import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory; import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory; import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory; import org.elasticsearch.indices.analysis.PreBuiltCharFilters; @@ -253,8 +250,8 @@ private static String toCamelCase(String s) { .put("turkishlowercase", LowerCaseTokenFilterFactory.class) .put("type", KeepTypesFilterFactory.class) .put("uppercase", UpperCaseTokenFilterFactory.class) - .put("worddelimiter", WordDelimiterTokenFilterFactory.class) - .put("worddelimitergraph", WordDelimiterGraphFilterFactory.class) + .put("worddelimiter", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. + .put("worddelimitergraph", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. .put("flattengraph", FlattenGraphTokenFilterFactory.class) // TODO: these tokenfilters are not yet exposed: useful? From e0cef658705b29633c67c51d621a38601ff80092 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 16 Mar 2017 11:31:24 -0400 Subject: [PATCH 04/28] Move some tests These tests check the interaction of analyzers and core components so they have to move to where the analyzers are available. --- .../indices/TransportAnalyzeActionTests.java | 92 +++--- .../index/analysis/AnalysisFactoryTests.java | 3 +- .../index/analysis/AnalysisRegistryTests.java | 13 +- .../highlight/HighlighterSearchIT.java | 154 ++-------- .../search/query/SearchQueryIT.java | 27 -- .../common/CommonAnalysisFactoryTests.java | 4 +- .../common/HighlighterWithAnalyzersTests.java | 143 +++++++++ .../common/QueryStringWithAnalyzersTests.java | 78 +++++ .../common/TransportAnalyzeActionTests.java | 275 ++++++++++++++++++ .../test/indices.analyze/10_analyze.yaml | 8 +- .../AnalysisFactoryTestCase.java | 6 +- 11 files changed, 587 insertions(+), 216 deletions(-) create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java rename {rest-api-spec/src/main => modules/analysis-common/src/test}/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml (97%) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index bcd7bba8d3808..2df9a6eacad09 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.admin.indices; +import org.apache.lucene.analysis.MockTokenFilter; +import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; @@ -27,18 +29,27 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; import java.util.List; +import java.util.Map; -import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +/** + * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the {@code common-analysis} module. + */ public class TransportAnalyzeActionTests extends ESTestCase { private IndexAnalyzers indexAnalyzers; @@ -53,23 +64,28 @@ public void setUp() throws Exception { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") - .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") - .put("index.analysis.tokenizer.trigram.type", "ngram") - .put("index.analysis.tokenizer.trigram.min_gram", 3) - .put("index.analysis.tokenizer.trigram.max_gram", 3) - .put("index.analysis.filter.synonym.type", "synonym") - .putArray("index.analysis.filter.synonym.synonyms", "kimchy => shay") - .put("index.analysis.filter.synonym.tokenizer", "trigram") - .put("index.analysis.filter.synonym.min_gram", 3) - .put("index.analysis.filter.synonym.max_gram", 3).build(); + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard") + .put("index.analysis.analyzer.custom_analyzer.filter", "mock").build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); environment = new Environment(settings); - registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry(); + AnalysisPlugin plugin = new AnalysisPlugin() { + class MockFactory extends AbstractTokenFilterFactory { + MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET); + } + } + + @Override + public Map> getTokenFilters() { + return singletonMap("mock", MockFactory::new); + } + }; + registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry(); indexAnalyzers = registry.build(idxSettings); } @@ -143,51 +159,43 @@ public void testFillsAttributes() throws IOException { } public void testWithIndexAnalyzers() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); request.text("the quick brown fox"); + request.analyzer("custom_analyzer"); - request.text("the qu1ck brown fox"); AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); + assertEquals(3, tokens.size()); + assertEquals("quick", tokens.get(0).getTerm()); + assertEquals("brown", tokens.get(1).getTerm()); + assertEquals("fox", tokens.get(2).getTerm()); - request.analyzer("whitespace"); - request.text("the qu1ck brown fox-dog"); + request.analyzer("standard"); analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); tokens = analyze.getTokens(); assertEquals(4, tokens.size()); - - request.analyzer("custom_analyzer"); - request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - tokens = analyze.getTokens(); - assertEquals(5, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("quick", tokens.get(1).getTerm()); + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals("fox", tokens.get(3).getTerm()); request.analyzer(null); - request.tokenizer("whitespace"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("wordDelimiter"); - request.text("the qu1ck brown fox-dog"); + request.tokenizer("standard"); analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); tokens = analyze.getTokens(); - assertEquals(5, tokens.size()); + assertEquals(4, tokens.size()); assertEquals("the", tokens.get(0).getTerm()); - assertEquals("qu1ck", tokens.get(1).getTerm()); + assertEquals("quick", tokens.get(1).getTerm()); assertEquals("brown", tokens.get(2).getTerm()); assertEquals("fox", tokens.get(3).getTerm()); - assertEquals("dog", tokens.get(4).getTerm()); - request.analyzer(null); - request.tokenizer("trigram"); - request.addTokenFilter("synonym"); - request.text("kimchy"); + request.addTokenFilter("mock"); analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); tokens = analyze.getTokens(); - assertEquals(2, tokens.size()); - assertEquals("sha", tokens.get(0).getTerm()); - assertEquals("hay", tokens.get(1).getTerm()); + assertEquals(3, tokens.size()); + assertEquals("quick", tokens.get(0).getTerm()); + assertEquals("brown", tokens.get(1).getTerm()); + assertEquals("fox", tokens.get(2).getTerm()); } public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java index 8c2f0ee1d51da..0a62e8c491588 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java @@ -22,6 +22,5 @@ import org.elasticsearch.AnalysisFactoryTestCase; public class AnalysisFactoryTests extends AnalysisFactoryTestCase { - // tests are inherited - // NOCOMMIT this should be dropped from core when there are no analyzers exposed there + // tests are inherited and nothing needs to be defined here } diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 6d161e6b4bd59..403035550f7cb 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -24,31 +24,26 @@ import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.util.CharArraySet; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.indices.analysis.PreBuiltAnalyzers; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; @@ -169,8 +164,8 @@ public void testConfigureCamelCaseTokenFilter() throws IOException { /* The snake_case version of the name should not filter out any stopwords while the * camelCase version will filter out English stopwords. */ AnalysisPlugin plugin = new AnalysisPlugin() { - class WordDelimiterFactory extends AbstractTokenFilterFactory { - public WordDelimiterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + class MockFactory extends AbstractTokenFilterFactory { + MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); } @@ -185,7 +180,7 @@ public TokenStream create(TokenStream tokenStream) { @Override public Map> getTokenFilters() { Map> filters = new HashMap<>(); - filters.put("mock", WordDelimiterFactory::new); + filters.put("mock", MockFactory::new); return filters; } }; diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java index 7db99ff3232ee..0e90ead4e5bb8 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase.highlight; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -100,6 +101,7 @@ import static org.hamcrest.Matchers.startsWith; public class HighlighterSearchIT extends ESIntegTestCase { + // TODO as we move analyzers out of the core we need to move some of these into HighlighterWithAnalyzersTests private static final String[] ALL_TYPES = new String[] {"plain", "postings", "fvh", "unified"}; private static final String[] UNIFIED_AND_NULL = new String[] {null, "unified"}; @@ -113,12 +115,11 @@ public void testHighlightingWithStoredKeyword() throws IOException { mappings.startObject(); mappings.startObject("type") .startObject("properties") - .startObject("text") - .field("type", "keyword") - .field("store", true) - .endObject() - .endObject() - .endObject(); + .startObject("text") + .field("type", "keyword") + .field("store", true) + .endObject() + .endObject().endObject(); mappings.endObject(); assertAcked(prepareCreate("test") .addMapping("type", mappings)); @@ -139,14 +140,13 @@ public void testHighlightingWithWildcardName() throws IOException { mappings.startObject(); mappings.startObject("type") .startObject("properties") - .startObject("text") - .field("type", "text") - .field("analyzer", "keyword") - .field("index_options", "offsets") - .field("term_vector", "with_positions_offsets") - .endObject() - .endObject() - .endObject(); + .startObject("text") + .field("type", "text") + .field("analyzer", "keyword") + .field("index_options", "offsets") + .field("term_vector", "with_positions_offsets") + .endObject() + .endObject().endObject(); mappings.endObject(); assertAcked(prepareCreate("test") .addMapping("type", mappings)); @@ -166,23 +166,22 @@ public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOExc mappings.startObject(); mappings.startObject("type") .startObject("_source") - .field("enabled", false) + .field("enabled", false) .endObject() .startObject("properties") - .startObject("unstored_field") - .field("index_options", "offsets") - .field("term_vector", "with_positions_offsets") - .field("type", "text") - .field("store", false) - .endObject() - .startObject("text") - .field("index_options", "offsets") - .field("term_vector", "with_positions_offsets") - .field("type", "text") - .field("store", true) - .endObject() - .endObject() - .endObject(); + .startObject("unstored_field") + .field("index_options", "offsets") + .field("term_vector", "with_positions_offsets") + .field("type", "text") + .field("store", false) + .endObject() + .startObject("text") + .field("index_options", "offsets") + .field("term_vector", "with_positions_offsets") + .field("type", "text") + .field("store", true) + .endObject() + .endObject().endObject(); mappings.endObject(); assertAcked(prepareCreate("test") .addMapping("type", mappings)); @@ -218,103 +217,6 @@ public void testHighTermFrequencyDoc() throws IOException { assertHighlight(search, 0, "name", 0, startsWith("abc abc abc abc")); } - public void testNgramHighlightingWithBrokenPositions() throws IOException { - assertAcked(prepareCreate("test") - .addMapping("test", jsonBuilder() - .startObject() - .startObject("test") - .startObject("properties") - .startObject("name") - .startObject("fields") - .startObject("autocomplete") - .field("type", "text") - .field("analyzer", "autocomplete") - .field("search_analyzer", "search_autocomplete") - .field("term_vector", "with_positions_offsets") - .endObject() - .endObject() - .field("type", "text") - .endObject() - .endObject() - .endObject() - .endObject()) - .setSettings(Settings.builder() - .put(indexSettings()) - .put("analysis.tokenizer.autocomplete.max_gram", 20) - .put("analysis.tokenizer.autocomplete.min_gram", 1) - .put("analysis.tokenizer.autocomplete.token_chars", "letter,digit") - .put("analysis.tokenizer.autocomplete.type", "nGram") - .put("analysis.filter.wordDelimiter.type", "word_delimiter") - .putArray("analysis.filter.wordDelimiter.type_table", - "& => ALPHANUM", "| => ALPHANUM", "! => ALPHANUM", - "? => ALPHANUM", ". => ALPHANUM", "- => ALPHANUM", "# => ALPHANUM", "% => ALPHANUM", - "+ => ALPHANUM", ", => ALPHANUM", "~ => ALPHANUM", ": => ALPHANUM", "/ => ALPHANUM", - "^ => ALPHANUM", "$ => ALPHANUM", "@ => ALPHANUM", ") => ALPHANUM", "( => ALPHANUM", - "] => ALPHANUM", "[ => ALPHANUM", "} => ALPHANUM", "{ => ALPHANUM") - - .put("analysis.filter.wordDelimiter.type.split_on_numerics", false) - .put("analysis.filter.wordDelimiter.generate_word_parts", true) - .put("analysis.filter.wordDelimiter.generate_number_parts", false) - .put("analysis.filter.wordDelimiter.catenate_words", true) - .put("analysis.filter.wordDelimiter.catenate_numbers", true) - .put("analysis.filter.wordDelimiter.catenate_all", false) - - .put("analysis.analyzer.autocomplete.tokenizer", "autocomplete") - .putArray("analysis.analyzer.autocomplete.filter", "lowercase", "wordDelimiter") - .put("analysis.analyzer.search_autocomplete.tokenizer", "whitespace") - .putArray("analysis.analyzer.search_autocomplete.filter", "lowercase", "wordDelimiter"))); - client().prepareIndex("test", "test", "1") - .setSource("name", "ARCOTEL Hotels Deutschland").get(); - refresh(); - SearchResponse search = client().prepareSearch("test").setTypes("test") - .setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)) - .highlighter(new HighlightBuilder().field("name.autocomplete")).execute().actionGet(); - assertHighlight(search, 0, "name.autocomplete", 0, equalTo("ARCOTEL Hotels Deutschland")); - } - - public void testMultiPhraseCutoff() throws IOException { - /* - * MultiPhraseQuery can literally kill an entire node if there are too many terms in the - * query. We cut off and extract terms if there are more than 16 terms in the query - */ - assertAcked(prepareCreate("test") - .addMapping("test", - "body", "type=text,analyzer=custom_analyzer,search_analyzer=custom_analyzer,term_vector=with_positions_offsets") - .setSettings( - Settings.builder().put(indexSettings()) - .put("analysis.filter.wordDelimiter.type", "word_delimiter") - .put("analysis.filter.wordDelimiter.type.split_on_numerics", false) - .put("analysis.filter.wordDelimiter.generate_word_parts", true) - .put("analysis.filter.wordDelimiter.generate_number_parts", true) - .put("analysis.filter.wordDelimiter.catenate_words", true) - .put("analysis.filter.wordDelimiter.catenate_numbers", true) - .put("analysis.filter.wordDelimiter.catenate_all", false) - .put("analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")) - ); - - ensureGreen(); - client().prepareIndex("test", "test", "1") - .setSource("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " - + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " - + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " - + "http://twitter.com this is a test for highlighting feature") - .get(); - refresh(); - SearchResponse search = client().prepareSearch().setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com ")) - .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); - assertHighlight(search, 0, "body", 0, startsWith("Test: http://www.facebook.com")); - search = client() - .prepareSearch() - .setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " - + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " - + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " - + "http://twitter.com this is a test for highlighting feature")) - .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); - assertHighlight(search, 0, "body", 0, equalTo("Test: http://www.facebook.com " - + "http://elasticsearch.org http://xing.com http://cnn.com http://quora.com")); - } - public void testNgramHighlighting() throws IOException { assertAcked(prepareCreate("test") .addMapping("test", diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 05dc973f9e24c..a30049c70d538 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1605,33 +1605,6 @@ public void testQueryStringWithSynonyms() throws IOException { assertHitCount(searchResponse, 2); } - // see #3898 - public void testCustomWordDelimiterQueryString() { - assertAcked(client().admin().indices().prepareCreate("test") - .setSettings("analysis.analyzer.my_analyzer.type", "custom", - "analysis.analyzer.my_analyzer.tokenizer", "whitespace", - "analysis.analyzer.my_analyzer.filter", "custom_word_delimiter", - "analysis.filter.custom_word_delimiter.type", "word_delimiter", - "analysis.filter.custom_word_delimiter.generate_word_parts", "true", - "analysis.filter.custom_word_delimiter.generate_number_parts", "false", - "analysis.filter.custom_word_delimiter.catenate_numbers", "true", - "analysis.filter.custom_word_delimiter.catenate_words", "false", - "analysis.filter.custom_word_delimiter.split_on_case_change", "false", - "analysis.filter.custom_word_delimiter.split_on_numerics", "false", - "analysis.filter.custom_word_delimiter.stem_english_possessive", "false") - .addMapping("type1", "field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer")); - - client().prepareIndex("test", "type1", "1").setSource("field1", "foo bar baz", "field2", "not needed").get(); - refresh(); - - SearchResponse response = client() - .prepareSearch("test") - .setQuery( - queryStringQuery("foo.baz").useDisMax(false).defaultOperator(Operator.AND) - .field("field1").field("field2")).get(); - assertHitCount(response, 1L); - } - // see #3797 public void testMultiMatchLenientIssue3797() { createIndex("test"); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index 6793278a42d51..fa546f3d753f6 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -34,7 +34,9 @@ protected Map> getTokenizers() { @Override protected Map> getTokenFilters() { Map> filters = new HashMap<>(super.getTokenFilters()); - filters.put("asciifolding", ASCIIFoldingTokenFilterFactory.class); + filters.put("asciifolding", ASCIIFoldingTokenFilterFactory.class); + filters.put("worddelimiter", WordDelimiterTokenFilterFactory.class); + filters.put("worddelimitergraph", WordDelimiterGraphTokenFilterFactory.class); return filters; } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java new file mode 100644 index 0000000000000..50b1c53a234f7 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -0,0 +1,143 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.test.ESIntegTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHighlight; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; + +public class HighlighterWithAnalyzersTests extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + public void testNgramHighlightingWithBrokenPositions() throws IOException { + assertAcked(prepareCreate("test") + .addMapping("test", jsonBuilder() + .startObject() + .startObject("test") + .startObject("properties") + .startObject("name") + .field("type", "text") + .startObject("fields") + .startObject("autocomplete") + .field("type", "text") + .field("analyzer", "autocomplete") + .field("search_analyzer", "search_autocomplete") + .field("term_vector", "with_positions_offsets") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject()) + .setSettings(Settings.builder() + .put(indexSettings()) + .put("analysis.tokenizer.autocomplete.max_gram", 20) + .put("analysis.tokenizer.autocomplete.min_gram", 1) + .put("analysis.tokenizer.autocomplete.token_chars", "letter,digit") + .put("analysis.tokenizer.autocomplete.type", "nGram") + .put("analysis.filter.wordDelimiter.type", "word_delimiter") + .putArray("analysis.filter.wordDelimiter.type_table", + "& => ALPHANUM", "| => ALPHANUM", "! => ALPHANUM", + "? => ALPHANUM", ". => ALPHANUM", "- => ALPHANUM", "# => ALPHANUM", "% => ALPHANUM", + "+ => ALPHANUM", ", => ALPHANUM", "~ => ALPHANUM", ": => ALPHANUM", "/ => ALPHANUM", + "^ => ALPHANUM", "$ => ALPHANUM", "@ => ALPHANUM", ") => ALPHANUM", "( => ALPHANUM", + "] => ALPHANUM", "[ => ALPHANUM", "} => ALPHANUM", "{ => ALPHANUM") + + .put("analysis.filter.wordDelimiter.type.split_on_numerics", false) + .put("analysis.filter.wordDelimiter.generate_word_parts", true) + .put("analysis.filter.wordDelimiter.generate_number_parts", false) + .put("analysis.filter.wordDelimiter.catenate_words", true) + .put("analysis.filter.wordDelimiter.catenate_numbers", true) + .put("analysis.filter.wordDelimiter.catenate_all", false) + + .put("analysis.analyzer.autocomplete.tokenizer", "autocomplete") + .putArray("analysis.analyzer.autocomplete.filter", "lowercase", "wordDelimiter") + .put("analysis.analyzer.search_autocomplete.tokenizer", "whitespace") + .putArray("analysis.analyzer.search_autocomplete.filter", "lowercase", "wordDelimiter"))); + client().prepareIndex("test", "test", "1") + .setSource("name", "ARCOTEL Hotels Deutschland").get(); + refresh(); + SearchResponse search = client().prepareSearch("test").setTypes("test") + .setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)) + .highlighter(new HighlightBuilder().field("name.autocomplete")).execute().actionGet(); + assertHighlight(search, 0, "name.autocomplete", 0, equalTo("ARCOTEL Hotels Deutschland")); + } + + public void testMultiPhraseCutoff() throws IOException { + /* + * MultiPhraseQuery can literally kill an entire node if there are too many terms in the + * query. We cut off and extract terms if there are more than 16 terms in the query + */ + assertAcked(prepareCreate("test") + .addMapping("test", + "body", "type=text,analyzer=custom_analyzer,search_analyzer=custom_analyzer,term_vector=with_positions_offsets") + .setSettings( + Settings.builder().put(indexSettings()) + .put("analysis.filter.wordDelimiter.type", "word_delimiter") + .put("analysis.filter.wordDelimiter.type.split_on_numerics", false) + .put("analysis.filter.wordDelimiter.generate_word_parts", true) + .put("analysis.filter.wordDelimiter.generate_number_parts", true) + .put("analysis.filter.wordDelimiter.catenate_words", true) + .put("analysis.filter.wordDelimiter.catenate_numbers", true) + .put("analysis.filter.wordDelimiter.catenate_all", false) + .put("analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")) + ); + + ensureGreen(); + client().prepareIndex("test", "test", "1") + .setSource("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " + + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " + + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " + + "http://twitter.com this is a test for highlighting feature") + .get(); + refresh(); + SearchResponse search = client().prepareSearch().setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com ")) + .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); + assertHighlight(search, 0, "body", 0, startsWith("Test: http://www.facebook.com")); + search = client() + .prepareSearch() + .setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " + + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " + + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " + + "http://twitter.com this is a test for highlighting feature")) + .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); + assertHighlight(search, 0, "body", 0, equalTo("Test: http://www.facebook.com " + + "http://elasticsearch.org http://xing.com http://cnn.com http://quora.com")); + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java new file mode 100644 index 0000000000000..63406ef5df915 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; + +public class QueryStringWithAnalyzersTests extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList(CommonAnalysisPlugin.class); + } + + @Override + protected int maximumNumberOfShards() { + return 7; + } + + @Override + protected int maximumNumberOfReplicas() { + return Math.min(2, cluster().numDataNodes() - 1); + } + + /** + * Validates that we properly split fields using the word delimiter filter in query_string. + */ + public void testCustomWordDelimiterQueryString() { + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings("analysis.analyzer.my_analyzer.type", "custom", + "analysis.analyzer.my_analyzer.tokenizer", "whitespace", + "analysis.analyzer.my_analyzer.filter", "custom_word_delimiter", + "analysis.filter.custom_word_delimiter.type", "word_delimiter", + "analysis.filter.custom_word_delimiter.generate_word_parts", "true", + "analysis.filter.custom_word_delimiter.generate_number_parts", "false", + "analysis.filter.custom_word_delimiter.catenate_numbers", "true", + "analysis.filter.custom_word_delimiter.catenate_words", "false", + "analysis.filter.custom_word_delimiter.split_on_case_change", "false", + "analysis.filter.custom_word_delimiter.split_on_numerics", "false", + "analysis.filter.custom_word_delimiter.stem_english_possessive", "false") + .addMapping("type1", "field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer")); + + client().prepareIndex("test", "type1", "1").setSource("field1", "foo bar baz", "field2", "not needed").get(); + refresh(); + + SearchResponse response = client() + .prepareSearch("test") + .setQuery( + queryStringQuery("foo.baz").useDisMax(false).defaultOperator(Operator.AND) + .field("field1").field("field2")).get(); + assertHitCount(response, 1L); + } +} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java new file mode 100644 index 0000000000000..96863a4fca4b3 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java @@ -0,0 +1,275 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.analysis.common; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.AllFieldMapper; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.singletonList; + +/** + * More "intense" version of a unit test with the same name that is in core. This one has access to the analyzers in this module. + */ +public class TransportAnalyzeActionTests extends ESTestCase { + private IndexAnalyzers indexAnalyzers; + private AnalysisRegistry registry; + private Environment environment; + + @Override + public void setUp() throws Exception { + super.setUp(); + Settings settings = Settings.builder().put( + Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + + Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") + .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .put("index.analysis.tokenizer.trigram.type", "ngram") + .put("index.analysis.tokenizer.trigram.min_gram", 3) + .put("index.analysis.tokenizer.trigram.max_gram", 3) + .put("index.analysis.filter.synonym.type", "synonym") + .putArray("index.analysis.filter.synonym.synonyms", "kimchy => shay") + .put("index.analysis.filter.synonym.tokenizer", "trigram") + .put("index.analysis.filter.synonym.min_gram", 3) + .put("index.analysis.filter.synonym.max_gram", 3).build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); + environment = new Environment(settings); + registry = new AnalysisModule(environment, singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry(); + indexAnalyzers = registry.build(idxSettings); + } + + public void testNoIndexAnalyzers() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the quick brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.addTokenFilter("lowercase"); + request.addTokenFilter("word_delimiter"); + request.text("the qu1ck brown fox"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, + environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.addCharFilter("html_strip"); + request.addTokenFilter("lowercase"); + request.addTokenFilter("word_delimiter"); + request.text("

the qu1ck brown fox

"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, + environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + assertEquals("brown", tokens.get(4).getTerm()); + assertEquals("fox", tokens.get(5).getTerm()); + } + + public void testFillsAttributes() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the 1 brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals(0, tokens.get(0).getStartOffset()); + assertEquals(3, tokens.get(0).getEndOffset()); + assertEquals(0, tokens.get(0).getPosition()); + assertEquals("", tokens.get(0).getType()); + + assertEquals("1", tokens.get(1).getTerm()); + assertEquals(4, tokens.get(1).getStartOffset()); + assertEquals(5, tokens.get(1).getEndOffset()); + assertEquals(1, tokens.get(1).getPosition()); + assertEquals("", tokens.get(1).getType()); + + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals(6, tokens.get(2).getStartOffset()); + assertEquals(11, tokens.get(2).getEndOffset()); + assertEquals(2, tokens.get(2).getPosition()); + assertEquals("", tokens.get(2).getType()); + + assertEquals("fox", tokens.get(3).getTerm()); + assertEquals(12, tokens.get(3).getStartOffset()); + assertEquals(15, tokens.get(3).getEndOffset()); + assertEquals(3, tokens.get(3).getPosition()); + assertEquals("", tokens.get(3).getType()); + } + + public void testWithIndexAnalyzers() throws IOException { + + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the quick brown fox"); + request.analyzer("custom_analyzer"); + request.text("the qu1ck brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer("whitespace"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer("custom_analyzer"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(5, tokens.size()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.addTokenFilter("lowercase"); + request.addTokenFilter("wordDelimiter"); + request.text("the qu1ck brown fox-dog"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(5, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("qu1ck", tokens.get(1).getTerm()); + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals("fox", tokens.get(3).getTerm()); + assertEquals("dog", tokens.get(4).getTerm()); + + request.analyzer(null); + request.tokenizer("trigram"); + request.addTokenFilter("synonym"); + request.text("kimchy"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(2, tokens.size()); + assertEquals("sha", tokens.get(0).getTerm()); + assertEquals("hay", tokens.get(1).getTerm()); + } + + public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .analyzer("custom_analyzer") + .text("the qu1ck brown fox-dog"), + AllFieldMapper.NAME, null, null, registry, environment)); + assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]"); + } + + public void testUnknown() throws IOException { + boolean notGlobal = randomBoolean(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .analyzer("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global analyzer [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global tokenizer under [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("whitespace") + .addTokenFilter("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global token filter under [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("whitespace") + .addTokenFilter("lowercase") + .addCharFilter("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global char filter under [foobar]"); + } + } + + public void testNonPreBuildTokenFilter() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.tokenizer("whitespace"); + request.addTokenFilter("min_hash"); + request.text("the quick brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + List tokens = analyze.getTokens(); + int default_hash_count = 1; + int default_bucket_size = 512; + int default_hash_set_size = 1; + assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size()); + + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml similarity index 97% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml rename to modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml index 268cd7812894f..e4997381556e2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml @@ -1,10 +1,6 @@ -# Will be performed before each test as a part of the test setup -# -setup: - - do: - ping: {} +# The rest tests for the indices.analyze action live in the common-analysis +# module because they depend on many analyzers not in core. ---- "Basic test": - do: indices.analyze: diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 2f723499bac06..ed6a10706e2d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -175,7 +175,7 @@ private static String toCamelCase(String s) { .put("apostrophe", ApostropheFilterFactory.class) .put("arabicnormalization", ArabicNormalizationFilterFactory.class) .put("arabicstem", ArabicStemTokenFilterFactory.class) - .put("asciifolding", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. + .put("asciifolding", Void.class) // TODO remove this when core no longer depends on analysis-common .put("brazilianstem", BrazilianStemTokenFilterFactory.class) .put("bulgarianstem", StemmerTokenFilterFactory.class) .put("cjkbigram", CJKBigramFilterFactory.class) @@ -250,8 +250,8 @@ private static String toCamelCase(String s) { .put("turkishlowercase", LowerCaseTokenFilterFactory.class) .put("type", KeepTypesFilterFactory.class) .put("uppercase", UpperCaseTokenFilterFactory.class) - .put("worddelimiter", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. - .put("worddelimitergraph", Void.class) // NOCOMMIT should this shouldn't be needed once we drop the dependency. + .put("worddelimiter", Void.class) // TODO remove this when core no longer depends on analysis-common + .put("worddelimitergraph", Void.class) // TODO remove this when core no longer depends on analysis-common .put("flattengraph", FlattenGraphTokenFilterFactory.class) // TODO: these tokenfilters are not yet exposed: useful? From 2310f3c4093310f1bf8cdefb1f745440142524b2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 16 Mar 2017 12:22:06 -0400 Subject: [PATCH 05/28] More tests --- ...aseWordDelimiterTokenFilterFactoryTestCase.java | 14 +++++++------- .../WordDelimiterGraphTokenFilterFactoryTests.java | 2 +- .../org/elasticsearch/AnalysisFactoryTestCase.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index c6a371fca9871..681b0c3b54222 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -45,7 +45,7 @@ public void testDefault() throws IOException { ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_word_delimiter.type", type) - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", @@ -61,7 +61,7 @@ public void testCatenateWords() throws IOException { .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"PowerShot", "500", "42", "wifi", "wifi", "4000", "j", "2", "se", "ONeil"}; @@ -76,7 +76,7 @@ public void testCatenateNumbers() throws IOException { .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") .put("index.analysis.filter.my_word_delimiter.catenate_numbers", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"Power", "Shot", "50042", "wi", "fi", "wi", "fi", "4000", "j", "2", @@ -93,7 +93,7 @@ public void testCatenateAll() throws IOException { .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"PowerShot", "50042", "wifi", "wifi4000", "j2se", "ONeil"}; @@ -107,7 +107,7 @@ public void testSplitOnCaseChange() throws IOException { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.split_on_case_change", "false") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; String[] expected = new String[]{"PowerShot"}; @@ -121,7 +121,7 @@ public void testPreserveOriginal() throws IOException { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"PowerShot", "Power", "Shot", "500-42", "500", "42", "wi-fi", "wi", "fi", @@ -136,7 +136,7 @@ public void testStemEnglishPossessive() throws IOException { .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.stem_english_possessive", "false") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", "4000", "j", "2", diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index a640553abd44f..a9602a164e031 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -63,7 +63,7 @@ public void testPartsAndCatenate() throws IOException { .put("index.analysis.filter.my_word_delimiter.type", type) .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") - .build()); + .build(), new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; int[] expectedIncr = new int[]{1, 0, 1}; diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index ed6a10706e2d6..07477a7eb0bbc 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -175,7 +175,7 @@ private static String toCamelCase(String s) { .put("apostrophe", ApostropheFilterFactory.class) .put("arabicnormalization", ArabicNormalizationFilterFactory.class) .put("arabicstem", ArabicStemTokenFilterFactory.class) - .put("asciifolding", Void.class) // TODO remove this when core no longer depends on analysis-common + .put("asciifolding", Void.class) // TODO remove this when core no longer depends on analysis-common .put("brazilianstem", BrazilianStemTokenFilterFactory.class) .put("bulgarianstem", StemmerTokenFilterFactory.class) .put("cjkbigram", CJKBigramFilterFactory.class) From dcbc7cf700808345099bc978e966c2a96349a1ce Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 12:56:00 -0400 Subject: [PATCH 06/28] Javadoc for breaksFastVectorHighlighter --- .../elasticsearch/index/analysis/TokenFilterFactory.java | 7 ++++++- .../fetch/subphase/highlight/FragmentBuilderHelper.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index 6456bd29258f7..fd8cbade54f7d 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -20,13 +20,18 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; +import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; public interface TokenFilterFactory { - String name(); TokenStream create(TokenStream tokenStream); + /** + * Does this analyzer mess up the {@link OffsetAttribute}s in such as way as to break the + * {@link FastVectorHighlighter}? If this is {@code true} then the + * {@linkplain FastVectorHighlighter} will attempt to work around the broken offsets. + */ default boolean breaksFastVectorHighlighter() { return false; } diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index 0d89e611d98ca..37971e6b48044 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -50,7 +50,7 @@ private FragmentBuilderHelper() { public static WeightedFragInfo fixWeightedFragInfo(FieldMapper mapper, Field[] values, WeightedFragInfo fragInfo) { assert fragInfo != null : "FragInfo must not be null"; assert mapper.fieldType().name().equals(values[0].name()) : "Expected FieldMapper for field " + values[0].name(); - if (!fragInfo.getSubInfos().isEmpty() && (containsBrokenAnalysis(mapper.fieldType().indexAnalyzer()))) { + if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(mapper.fieldType().indexAnalyzer())) { /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort * the fragments based on their offsets rather than using soley the positions as it is done in From c20aca47168208ed313e897a2ac0376d9485cfae Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 12:59:57 -0400 Subject: [PATCH 07/28] Line length --- .../resources/checkstyle_suppressions.xml | 7 -- .../ASCIIFoldingTokenFilterFactory.java | 13 ++- .../WordDelimiterGraphTokenFilterFactory.java | 24 ++-- .../ASCIIFoldingTokenFilterFactoryTests.java | 25 ++-- ...rdDelimiterTokenFilterFactoryTestCase.java | 109 ++++++++++-------- 5 files changed, 99 insertions(+), 79 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index c95ad03f9ac60..b155e77bab820 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -1095,7 +1095,6 @@ - @@ -1224,8 +1223,6 @@ - - @@ -2685,11 +2682,9 @@ - - @@ -2708,8 +2703,6 @@ - - diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java index 3991c7f44bd1f..f8e0c7383a09b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactory.java @@ -32,16 +32,19 @@ /** * Factory for ASCIIFoldingFilter. */ -public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent { - public static ParseField PRESERVE_ORIGINAL = new ParseField("preserve_original"); - public static boolean DEFAULT_PRESERVE_ORIGINAL = false; +public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory + implements MultiTermAwareComponent { + public static final ParseField PRESERVE_ORIGINAL = new ParseField("preserve_original"); + public static final boolean DEFAULT_PRESERVE_ORIGINAL = false; private final boolean preserveOriginal; - public ASCIIFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) { + public ASCIIFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, + String name, Settings settings) { super(indexSettings, name, settings); preserveOriginal = settings.getAsBooleanLenientForPreEs6Indices( - indexSettings.getIndexVersionCreated(), PRESERVE_ORIGINAL.getPreferredName(), DEFAULT_PRESERVE_ORIGINAL, deprecationLogger); + indexSettings.getIndexVersionCreated(), PRESERVE_ORIGINAL.getPreferredName(), + DEFAULT_PRESERVE_ORIGINAL, deprecationLogger); } @Override diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java index 0d4a5a2eed1a2..1613339853117 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactory.java @@ -32,15 +32,15 @@ import java.util.List; import java.util.Set; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_ALL; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_NUMBERS; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.CATENATE_WORDS; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.GENERATE_NUMBER_PARTS; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.GENERATE_WORD_PARTS; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.PRESERVE_ORIGINAL; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_CASE_CHANGE; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.SPLIT_ON_NUMERICS; -import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.STEM_ENGLISH_POSSESSIVE; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.CATENATE_ALL; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.CATENATE_NUMBERS; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.CATENATE_WORDS; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.GENERATE_WORD_PARTS; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.PRESERVE_ORIGINAL; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.SPLIT_ON_NUMERICS; +import static org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE; import static org.elasticsearch.analysis.common.WordDelimiterTokenFilterFactory.parseTypes; public class WordDelimiterGraphTokenFilterFactory extends AbstractTokenFilterFactory { @@ -49,7 +49,8 @@ public class WordDelimiterGraphTokenFilterFactory extends AbstractTokenFilterFac private final int flags; private final CharArraySet protoWords; - public WordDelimiterGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + public WordDelimiterGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, + String name, Settings settings) { super(indexSettings, name, settings); // Sample Format for the type table: @@ -84,7 +85,8 @@ public WordDelimiterGraphTokenFilterFactory(IndexSettings indexSettings, Environ // If set, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil" flags |= getFlag(STEM_ENGLISH_POSSESSIVE, settings, "stem_english_possessive", true); // If not null is the set of tokens to protect from being delimited - Set protectedWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, "protected_words"); + Set protectedWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), + settings, "protected_words"); this.protoWords = protectedWords == null ? null : CharArraySet.copy(protectedWords); this.flags = flags; } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java index 813311c0857b9..22ac081011ff4 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ASCIIFoldingTokenFilterFactoryTests.java @@ -34,10 +34,12 @@ public class ASCIIFoldingTokenFilterFactoryTests extends ESTokenStreamTestCase { public void testDefault() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding"); String source = "Ansprüche"; String[] expected = new String[]{"Anspruche"}; @@ -47,11 +49,13 @@ public void testDefault() throws IOException { } public void testPreserveOriginal() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") - .put("index.analysis.filter.my_ascii_folding.preserve_original", true) - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_ascii_folding.type", "asciifolding") + .put("index.analysis.filter.my_ascii_folding.preserve_original", true) + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_ascii_folding"); String source = "Ansprüche"; String[] expected = new String[]{"Anspruche", "Ansprüche"}; @@ -60,7 +64,8 @@ public void testPreserveOriginal() throws IOException { assertTokenStreamContents(tokenFilter.create(tokenizer), expected); // but the multi-term aware component still emits a single token - tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter).getMultiTermComponent(); + tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter) + .getMultiTermComponent(); tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); expected = new String[]{"Anspruche"}; diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java index 681b0c3b54222..ce6d0403c0738 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/BaseWordDelimiterTokenFilterFactoryTestCase.java @@ -32,7 +32,8 @@ import java.io.StringReader; /** - * Base class to test {@link WordDelimiterTokenFilterFactory} and {@link WordDelimiterGraphTokenFilterFactory} + * Base class to test {@link WordDelimiterTokenFilterFactory} and + * {@link WordDelimiterGraphTokenFilterFactory}. */ public abstract class BaseWordDelimiterTokenFilterFactoryTestCase extends ESTokenStreamTestCase { final String type; @@ -42,10 +43,12 @@ public BaseWordDelimiterTokenFilterFactoryTestCase(String type) { } public void testDefault() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", @@ -56,44 +59,51 @@ public void testDefault() throws IOException { } public void testCatenateWords() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") - .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") + .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; - String[] expected = new String[]{"PowerShot", "500", "42", "wifi", "wifi", "4000", "j", "2", "se", "ONeil"}; + String[] expected = new String[] { "PowerShot", "500", "42", "wifi", "wifi", "4000", "j", + "2", "se", "ONeil" }; Tokenizer tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } public void testCatenateNumbers() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") - .put("index.analysis.filter.my_word_delimiter.catenate_numbers", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") + .put("index.analysis.filter.my_word_delimiter.catenate_numbers", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; - String[] expected = new String[]{"Power", "Shot", "50042", "wi", "fi", "wi", "fi", "4000", "j", "2", - "se", "O", "Neil"}; + String[] expected = new String[] { "Power", "Shot", "50042", "wi", "fi", "wi", "fi", "4000", + "j", "2", "se", "O", "Neil" }; Tokenizer tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } public void testCatenateAll() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") - .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") - .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "false") + .put("index.analysis.filter.my_word_delimiter.generate_number_parts", "false") + .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; String[] expected = new String[]{"PowerShot", "50042", "wifi", "wifi4000", "j2se", "ONeil"}; @@ -103,11 +113,13 @@ public void testCatenateAll() throws IOException { } public void testSplitOnCaseChange() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.split_on_case_change", "false") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.split_on_case_change", "false") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; String[] expected = new String[]{"PowerShot"}; @@ -117,30 +129,35 @@ public void testSplitOnCaseChange() throws IOException { } public void testPreserveOriginal() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; - String[] expected = new String[]{"PowerShot", "Power", "Shot", "500-42", "500", "42", "wi-fi", "wi", "fi", - "wi-fi-4000", "wi", "fi", "4000", "j2se", "j", "2", "se", "O'Neil's", "O", "Neil"}; + String[] expected = new String[] { "PowerShot", "Power", "Shot", "500-42", "500", "42", + "wi-fi", "wi", "fi", "wi-fi-4000", "wi", "fi", "4000", "j2se", "j", "2", "se", + "O'Neil's", "O", "Neil" }; Tokenizer tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); } public void testStemEnglishPossessive() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.stem_english_possessive", "false") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.stem_english_possessive", "false") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; - String[] expected = new String[]{"Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", "4000", "j", "2", - "se", "O", "Neil", "s"}; + String[] expected = new String[] { "Power", "Shot", "500", "42", "wi", "fi", "wi", "fi", + "4000", "j", "2", "se", "O", "Neil", "s" }; Tokenizer tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); assertTokenStreamContents(tokenFilter.create(tokenizer), expected); From a09c37f9c274c69a70817615539315136508f75b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:07:03 -0400 Subject: [PATCH 08/28] Import --- .../org/elasticsearch/index/analysis/TokenFilterFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java index fd8cbade54f7d..c90138d7a2375 100644 --- a/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java +++ b/core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; public interface TokenFilterFactory { From c92a6a8322b8b59324ddc7f7e4be6b4129850d91 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:45:03 -0400 Subject: [PATCH 09/28] Line length --- .../common/WordDelimiterTokenFilterFactory.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java index 3bf3cf384bc70..8c38beb8f8b7b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactory.java @@ -54,7 +54,8 @@ public class WordDelimiterTokenFilterFactory extends AbstractTokenFilterFactory private final int flags; private final CharArraySet protoWords; - public WordDelimiterTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + public WordDelimiterTokenFilterFactory(IndexSettings indexSettings, Environment env, + String name, Settings settings) { super(indexSettings, name, settings); // Sample Format for the type table: @@ -89,7 +90,8 @@ public WordDelimiterTokenFilterFactory(IndexSettings indexSettings, Environment // If set, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil" flags |= getFlag(STEM_ENGLISH_POSSESSIVE, settings, "stem_english_possessive", true); // If not null is the set of tokens to protect from being delimited - Set protectedWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, "protected_words"); + Set protectedWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), + settings, "protected_words"); this.protoWords = protectedWords == null ? null : CharArraySet.copy(protectedWords); this.flags = flags; } @@ -103,7 +105,8 @@ public TokenStream create(TokenStream tokenStream) { } public int getFlag(int flag, Settings settings, String key, boolean defaultValue) { - if (settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), key, defaultValue, deprecationLogger)) { + if (settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), + key, defaultValue, deprecationLogger)) { return flag; } return 0; @@ -124,14 +127,16 @@ static byte[] parseTypes(Collection rules) { String lhs = parseString(m.group(1).trim()); Byte rhs = parseType(m.group(2).trim()); if (lhs.length() != 1) - throw new RuntimeException("Invalid Mapping Rule : [" + rule + "]. Only a single character is allowed."); + throw new RuntimeException("Invalid Mapping Rule : [" + + rule + "]. Only a single character is allowed."); if (rhs == null) throw new RuntimeException("Invalid Mapping Rule : [" + rule + "]. Illegal type."); typeMap.put(lhs.charAt(0), rhs); } // ensure the table is always at least as big as DEFAULT_WORD_DELIM_TABLE for performance - byte types[] = new byte[Math.max(typeMap.lastKey() + 1, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE.length)]; + byte types[] = new byte[Math.max( + typeMap.lastKey() + 1, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE.length)]; for (int i = 0; i < types.length; i++) types[i] = WordDelimiterIterator.getType(i); for (Map.Entry mapping : typeMap.entrySet()) From 39df4c9c0b0476f7fcf17e26e0f5712358788306 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:46:02 -0400 Subject: [PATCH 10/28] line length --- ...DelimiterGraphTokenFilterFactoryTests.java | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index a9602a164e031..bd7ff2f0c0188 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.analysis.common; - import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.elasticsearch.common.settings.Settings; @@ -30,40 +29,49 @@ import java.io.IOException; import java.io.StringReader; -public class WordDelimiterGraphTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase { +public class WordDelimiterGraphTokenFilterFactoryTests + extends BaseWordDelimiterTokenFilterFactoryTestCase { public WordDelimiterGraphTokenFilterFactoryTests() { super("word_delimiter_graph"); } public void testMultiTerms() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") - .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.catenate_all", "true") + .put("index.analysis.filter.my_word_delimiter.preserve_original", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's"; - String[] expected = new String[]{"PowerShot", "PowerShot", "Power", "Shot", "50042", "500-42", "500", "42", - "wifi", "wi-fi", "wi", "fi", "wifi4000", "wi-fi-4000", "wi", "fi", "4000", "j2se", "j2se", "j", "2", "se", - "ONeil", "O'Neil's", "O", "Neil" }; + String[] expected = new String[] { "PowerShot", "PowerShot", "Power", "Shot", "50042", + "500-42", "500", "42", "wifi", "wi-fi", "wi", "fi", "wifi4000", "wi-fi-4000", "wi", + "fi", "4000", "j2se", "j2se", "j", "2", "se", "ONeil", "O'Neil's", "O", "Neil" }; Tokenizer tokenizer = new WhitespaceTokenizer(); tokenizer.setReader(new StringReader(source)); - int[] expectedIncr = new int[]{1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1}; - int[] expectedPosLen = new int[]{2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 3, 3, 1, 1, 1, 3, 3, 1, 1, 1, 2, 2, 1, 1}; + int[] expectedIncr = new int[] { 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, + 1, 1, 1, 0, 0, 1 }; + int[] expectedPosLen = new int[] { 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 3, 3, 1, 1, 1, 3, 3, + 1, 1, 1, 2, 2, 1, 1 }; assertTokenStreamContents(tokenFilter.create(tokenizer), expected, null, null, null, expectedIncr, expectedPosLen, null); } - /** Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power */ + /** + * Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power + */ public void testPartsAndCatenate() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") - .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") + .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; int[] expectedIncr = new int[]{1, 0, 1}; From 1ba49a6500e53e621470341ec862aea2916db58f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:48:47 -0400 Subject: [PATCH 11/28] Line length --- .../common/TransportAnalyzeActionTests.java | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java index 96863a4fca4b3..e0dc97f3611be 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java @@ -59,9 +59,11 @@ public void setUp() throws Exception { .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .putArray("index.analysis.analyzer.custom_analyzer.filter", + "lowercase", "wordDelimiter") .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter") + .putArray("index.analysis.analyzer.custom_analyzer.filter", + "lowercase", "wordDelimiter") .put("index.analysis.tokenizer.trigram.type", "ngram") .put("index.analysis.tokenizer.trigram.min_gram", 3) .put("index.analysis.tokenizer.trigram.max_gram", 3) @@ -72,7 +74,8 @@ public void setUp() throws Exception { .put("index.analysis.filter.synonym.max_gram", 3).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); environment = new Environment(settings); - registry = new AnalysisModule(environment, singletonList(new CommonAnalysisPlugin())).getAnalysisRegistry(); + registry = new AnalysisModule(environment, singletonList(new CommonAnalysisPlugin())) + .getAnalysisRegistry(); indexAnalyzers = registry.build(idxSettings); } @@ -80,7 +83,8 @@ public void testNoIndexAnalyzers() throws IOException { AnalyzeRequest request = new AnalyzeRequest(); request.analyzer("standard"); request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + null, registry, environment); List tokens = analyze.getTokens(); assertEquals(4, tokens.size()); @@ -89,8 +93,8 @@ public void testNoIndexAnalyzers() throws IOException { request.addTokenFilter("lowercase"); request.addTokenFilter("word_delimiter"); request.text("the qu1ck brown fox"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, - environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + randomBoolean() ? indexAnalyzers : null, registry, environment); tokens = analyze.getTokens(); assertEquals(6, tokens.size()); assertEquals("qu", tokens.get(1).getTerm()); @@ -103,8 +107,8 @@ public void testNoIndexAnalyzers() throws IOException { request.addTokenFilter("lowercase"); request.addTokenFilter("word_delimiter"); request.text("

the qu1ck brown fox

"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, - environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + randomBoolean() ? indexAnalyzers : null, registry, environment); tokens = analyze.getTokens(); assertEquals(6, tokens.size()); assertEquals("the", tokens.get(0).getTerm()); @@ -119,7 +123,8 @@ public void testFillsAttributes() throws IOException { AnalyzeRequest request = new AnalyzeRequest(); request.analyzer("standard"); request.text("the 1 brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + null, registry, environment); List tokens = analyze.getTokens(); assertEquals(4, tokens.size()); assertEquals("the", tokens.get(0).getTerm()); @@ -154,19 +159,22 @@ public void testWithIndexAnalyzers() throws IOException { request.text("the quick brown fox"); request.analyzer("custom_analyzer"); request.text("the qu1ck brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + indexAnalyzers, registry, environment); List tokens = analyze.getTokens(); assertEquals(4, tokens.size()); request.analyzer("whitespace"); request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, + registry, environment); tokens = analyze.getTokens(); assertEquals(4, tokens.size()); request.analyzer("custom_analyzer"); request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, + registry, environment); tokens = analyze.getTokens(); assertEquals(5, tokens.size()); @@ -175,7 +183,8 @@ public void testWithIndexAnalyzers() throws IOException { request.addTokenFilter("lowercase"); request.addTokenFilter("wordDelimiter"); request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, + registry, environment); tokens = analyze.getTokens(); assertEquals(5, tokens.size()); assertEquals("the", tokens.get(0).getTerm()); @@ -188,7 +197,8 @@ public void testWithIndexAnalyzers() throws IOException { request.tokenizer("trigram"); request.addTokenFilter("synonym"); request.text("kimchy"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, + registry, environment); tokens = analyze.getTokens(); assertEquals(2, tokens.size()); assertEquals("sha", tokens.get(0).getTerm()); @@ -212,7 +222,8 @@ public void testUnknown() throws IOException { new AnalyzeRequest() .analyzer("foobar") .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, + registry, environment)); if (notGlobal) { assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); } else { @@ -224,7 +235,8 @@ public void testUnknown() throws IOException { new AnalyzeRequest() .tokenizer("foobar") .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, + registry, environment)); if (notGlobal) { assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); } else { @@ -237,7 +249,8 @@ public void testUnknown() throws IOException { .tokenizer("whitespace") .addTokenFilter("foobar") .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, + registry, environment)); if (notGlobal) { assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); } else { @@ -251,7 +264,8 @@ public void testUnknown() throws IOException { .addTokenFilter("lowercase") .addCharFilter("foobar") .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, + registry, environment)); if (notGlobal) { assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); } else { @@ -264,12 +278,13 @@ public void testNonPreBuildTokenFilter() throws IOException { request.tokenizer("whitespace"); request.addTokenFilter("min_hash"); request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, + indexAnalyzers, registry, environment); List tokens = analyze.getTokens(); int default_hash_count = 1; int default_bucket_size = 512; int default_hash_set_size = 1; - assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size()); - + assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, + tokens.size()); } } From 1caf03f3b11c9c243a1e52e07f11480451e68822 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:54:55 -0400 Subject: [PATCH 12/28] Line length --- .../indices/TransportAnalyzeActionTests.java | 3 +- .../common/HighlighterWithAnalyzersTests.java | 57 +++++++++++-------- .../common/QueryStringWithAnalyzersTests.java | 8 ++- .../WordDelimiterTokenFilterFactoryTests.java | 18 +++--- 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index 2df9a6eacad09..b984e597384be 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -48,7 +48,8 @@ import static java.util.Collections.singletonMap; /** - * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the {@code common-analysis} module. + * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the + * {@code common-analysis} module. */ public class TransportAnalyzeActionTests extends ESTestCase { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java index 50b1c53a234f7..c022d5c85acbb 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/HighlighterWithAnalyzersTests.java @@ -73,11 +73,13 @@ public void testNgramHighlightingWithBrokenPositions() throws IOException { .put("analysis.filter.wordDelimiter.type", "word_delimiter") .putArray("analysis.filter.wordDelimiter.type_table", "& => ALPHANUM", "| => ALPHANUM", "! => ALPHANUM", - "? => ALPHANUM", ". => ALPHANUM", "- => ALPHANUM", "# => ALPHANUM", "% => ALPHANUM", - "+ => ALPHANUM", ", => ALPHANUM", "~ => ALPHANUM", ": => ALPHANUM", "/ => ALPHANUM", - "^ => ALPHANUM", "$ => ALPHANUM", "@ => ALPHANUM", ") => ALPHANUM", "( => ALPHANUM", - "] => ALPHANUM", "[ => ALPHANUM", "} => ALPHANUM", "{ => ALPHANUM") - + "? => ALPHANUM", ". => ALPHANUM", "- => ALPHANUM", + "# => ALPHANUM", "% => ALPHANUM", "+ => ALPHANUM", + ", => ALPHANUM", "~ => ALPHANUM", ": => ALPHANUM", + "/ => ALPHANUM", "^ => ALPHANUM", "$ => ALPHANUM", + "@ => ALPHANUM", ") => ALPHANUM", "( => ALPHANUM", + "] => ALPHANUM", "[ => ALPHANUM", "} => ALPHANUM", + "{ => ALPHANUM") .put("analysis.filter.wordDelimiter.type.split_on_numerics", false) .put("analysis.filter.wordDelimiter.generate_word_parts", true) .put("analysis.filter.wordDelimiter.generate_number_parts", false) @@ -86,16 +88,19 @@ public void testNgramHighlightingWithBrokenPositions() throws IOException { .put("analysis.filter.wordDelimiter.catenate_all", false) .put("analysis.analyzer.autocomplete.tokenizer", "autocomplete") - .putArray("analysis.analyzer.autocomplete.filter", "lowercase", "wordDelimiter") + .putArray("analysis.analyzer.autocomplete.filter", + "lowercase", "wordDelimiter") .put("analysis.analyzer.search_autocomplete.tokenizer", "whitespace") - .putArray("analysis.analyzer.search_autocomplete.filter", "lowercase", "wordDelimiter"))); + .putArray("analysis.analyzer.search_autocomplete.filter", + "lowercase", "wordDelimiter"))); client().prepareIndex("test", "test", "1") .setSource("name", "ARCOTEL Hotels Deutschland").get(); refresh(); SearchResponse search = client().prepareSearch("test").setTypes("test") .setQuery(matchQuery("name.autocomplete", "deut tel").operator(Operator.OR)) - .highlighter(new HighlightBuilder().field("name.autocomplete")).execute().actionGet(); - assertHighlight(search, 0, "name.autocomplete", 0, equalTo("ARCOTEL Hotels Deutschland")); + .highlighter(new HighlightBuilder().field("name.autocomplete")).get(); + assertHighlight(search, 0, "name.autocomplete", 0, + equalTo("ARCOTEL Hotels Deutschland")); } public void testMultiPhraseCutoff() throws IOException { @@ -104,8 +109,8 @@ public void testMultiPhraseCutoff() throws IOException { * query. We cut off and extract terms if there are more than 16 terms in the query */ assertAcked(prepareCreate("test") - .addMapping("test", - "body", "type=text,analyzer=custom_analyzer,search_analyzer=custom_analyzer,term_vector=with_positions_offsets") + .addMapping("test", "body", "type=text,analyzer=custom_analyzer," + + "search_analyzer=custom_analyzer,term_vector=with_positions_offsets") .setSettings( Settings.builder().put(indexSettings()) .put("analysis.filter.wordDelimiter.type", "word_delimiter") @@ -116,28 +121,34 @@ public void testMultiPhraseCutoff() throws IOException { .put("analysis.filter.wordDelimiter.catenate_numbers", true) .put("analysis.filter.wordDelimiter.catenate_all", false) .put("analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")) + .putArray("analysis.analyzer.custom_analyzer.filter", + "lowercase", "wordDelimiter")) ); ensureGreen(); client().prepareIndex("test", "test", "1") - .setSource("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " - + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " - + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " + .setSource("body", "Test: http://www.facebook.com http://elasticsearch.org " + + "http://xing.com http://cnn.com http://quora.com http://twitter.com this is " + + "a test for highlighting feature Test: http://www.facebook.com " + + "http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " + "http://twitter.com this is a test for highlighting feature") .get(); refresh(); - SearchResponse search = client().prepareSearch().setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com ")) - .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); + SearchResponse search = client().prepareSearch() + .setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com ")) + .highlighter(new HighlightBuilder().field("body")).get(); assertHighlight(search, 0, "body", 0, startsWith("Test: http://www.facebook.com")); search = client() .prepareSearch() - .setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com http://elasticsearch.org http://xing.com " - + "http://cnn.com http://quora.com http://twitter.com this is a test for highlighting feature Test: " - + "http://www.facebook.com http://elasticsearch.org http://xing.com http://cnn.com http://quora.com " - + "http://twitter.com this is a test for highlighting feature")) + .setQuery(matchPhraseQuery("body", "Test: http://www.facebook.com " + + "http://elasticsearch.org http://xing.com http://cnn.com " + + "http://quora.com http://twitter.com this is a test for highlighting " + + "feature Test: http://www.facebook.com http://elasticsearch.org " + + "http://xing.com http://cnn.com http://quora.com http://twitter.com this " + + "is a test for highlighting feature")) .highlighter(new HighlightBuilder().field("body")).execute().actionGet(); - assertHighlight(search, 0, "body", 0, equalTo("Test: http://www.facebook.com " - + "http://elasticsearch.org http://xing.com http://cnn.com http://quora.com")); + assertHighlight(search, 0, "body", 0, equalTo("Test: " + + "http://www.facebook.com http://elasticsearch.org " + + "http://xing.com http://cnn.com http://quora.com")); } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java index 63406ef5df915..47b58b867e336 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java @@ -63,9 +63,13 @@ public void testCustomWordDelimiterQueryString() { "analysis.filter.custom_word_delimiter.split_on_case_change", "false", "analysis.filter.custom_word_delimiter.split_on_numerics", "false", "analysis.filter.custom_word_delimiter.stem_english_possessive", "false") - .addMapping("type1", "field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer")); + .addMapping("type1", + "field1", "type=text,analyzer=my_analyzer", + "field2", "type=text,analyzer=my_analyzer")); - client().prepareIndex("test", "type1", "1").setSource("field1", "foo bar baz", "field2", "not needed").get(); + client().prepareIndex("test", "type1", "1").setSource( + "field1", "foo bar baz", + "field2", "not needed").get(); refresh(); SearchResponse response = client() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java index 44efe4e826ef5..3b5fb9dcad4c9 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java @@ -35,14 +35,18 @@ public WordDelimiterTokenFilterFactoryTests() { super("word_delimiter"); } - /** Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power */ + /** + * Correct offset order when doing both parts and concatenation: PowerShot is a synonym of Power + */ public void testPartsAndCatenate() throws IOException { - ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) - .put("index.analysis.filter.my_word_delimiter.type", type) - .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") - .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") - .build(), new CommonAnalysisPlugin()); + ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings( + Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put("index.analysis.filter.my_word_delimiter.type", type) + .put("index.analysis.filter.my_word_delimiter.catenate_words", "true") + .put("index.analysis.filter.my_word_delimiter.generate_word_parts", "true") + .build(), + new CommonAnalysisPlugin()); TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter"); String source = "PowerShot"; String[] expected = new String[]{"Power", "PowerShot", "Shot" }; From 16102ffbe2784a9c457033b5c8ae0258aa7f54a4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:57:29 -0400 Subject: [PATCH 13/28] Line length --- .../analysis/common/TransportAnalyzeActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java index e0dc97f3611be..c7d0abceed3db 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java @@ -40,7 +40,8 @@ import static java.util.Collections.singletonList; /** - * More "intense" version of a unit test with the same name that is in core. This one has access to the analyzers in this module. + * More "intense" version of a unit test with the same name that is in core. This one has access to + * the analyzers in this module. */ public class TransportAnalyzeActionTests extends ESTestCase { private IndexAnalyzers indexAnalyzers; From 9ee485f1ff9b3244c411cc07e5d9b2880a6f568d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 13:58:11 -0400 Subject: [PATCH 14/28] line length --- .../analysis/common/WordDelimiterTokenFilterFactoryTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java index 3b5fb9dcad4c9..78c4f1485aa43 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterTokenFilterFactoryTests.java @@ -30,7 +30,8 @@ import java.io.IOException; import java.io.StringReader; -public class WordDelimiterTokenFilterFactoryTests extends BaseWordDelimiterTokenFilterFactoryTestCase { +public class WordDelimiterTokenFilterFactoryTests + extends BaseWordDelimiterTokenFilterFactoryTestCase { public WordDelimiterTokenFilterFactoryTests() { super("word_delimiter"); } From c98bcc4cca26875670a5838c483acdf54a1d46c7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 14:03:36 -0400 Subject: [PATCH 15/28] Line length --- .../src/main/resources/checkstyle_suppressions.xml | 1 - .../index/analysis/AnalysisTestsHelper.java | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index b155e77bab820..f81d8d32946a3 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -2684,7 +2684,6 @@ - diff --git a/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java index d55ea8a39d9b0..d75a894d0737c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java @@ -35,7 +35,8 @@ public class AnalysisTestsHelper { - public static ESTestCase.TestAnalysis createTestAnalysisFromClassPath(Path baseDir, String resource) throws IOException { + public static ESTestCase.TestAnalysis createTestAnalysisFromClassPath(Path baseDir, + String resource) throws IOException { Settings settings = Settings.builder() .loadFromStream(resource, AnalysisTestsHelper.class.getResourceAsStream(resource)) .put(Environment.PATH_HOME_SETTING.getKey(), baseDir.toString()) @@ -47,10 +48,13 @@ public static ESTestCase.TestAnalysis createTestAnalysisFromClassPath(Path baseD public static ESTestCase.TestAnalysis createTestAnalysisFromSettings( Settings settings, AnalysisPlugin... plugins) throws IOException { if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) { - settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + settings = Settings.builder().put(settings) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); } IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings); - AnalysisRegistry analysisRegistry = new AnalysisModule(new Environment(settings), Arrays.asList(plugins)).getAnalysisRegistry(); + AnalysisRegistry analysisRegistry = + new AnalysisModule(new Environment(settings), Arrays.asList(plugins)) + .getAnalysisRegistry(); return new ESTestCase.TestAnalysis(analysisRegistry.build(indexSettings), analysisRegistry.buildTokenFilterFactories(indexSettings), analysisRegistry.buildTokenizerFactories(indexSettings), From 04512d29c480629405bdfb22b74a9ef33f3b2a6e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 14:39:38 -0400 Subject: [PATCH 16/28] Explain tests a bit more --- .../action/admin/indices/TransportAnalyzeActionTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java index b984e597384be..57a83b2c68081 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -162,7 +162,6 @@ public void testFillsAttributes() throws IOException { public void testWithIndexAnalyzers() throws IOException { AnalyzeRequest request = new AnalyzeRequest(); request.text("the quick brown fox"); - request.analyzer("custom_analyzer"); AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); List tokens = analyze.getTokens(); @@ -180,6 +179,7 @@ public void testWithIndexAnalyzers() throws IOException { assertEquals("brown", tokens.get(2).getTerm()); assertEquals("fox", tokens.get(3).getTerm()); + // Switch the analyzer out for just a tokenizer request.analyzer(null); request.tokenizer("standard"); analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); @@ -190,6 +190,7 @@ public void testWithIndexAnalyzers() throws IOException { assertEquals("brown", tokens.get(2).getTerm()); assertEquals("fox", tokens.get(3).getTerm()); + // Now try applying our token filter request.addTokenFilter("mock"); analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); tokens = analyze.getTokens(); From efb639226e9df0e2b31614c5b7b52f0a052d98e4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 14:55:32 -0400 Subject: [PATCH 17/28] Explain funny test thing --- .../common/CommonAnalysisFactoryTests.java | 38 +++++++++++++++++++ .../AnalysisFactoryTestCase.java | 15 ++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java index fa546f3d753f6..886dad37b56f1 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java @@ -22,8 +22,12 @@ import org.elasticsearch.AnalysisFactoryTestCase; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; + public class CommonAnalysisFactoryTests extends AnalysisFactoryTestCase { @Override protected Map> getTokenizers() { @@ -45,4 +49,38 @@ protected Map> getCharFilters() { Map> filters = new HashMap<>(super.getCharFilters()); return filters; } + + /** + * Fails if a tokenizer is marked in the superclass with {@link MovedToAnalysisCommon} but + * hasn't been marked in this class with its proper factory. + */ + public void testAllTokenizersMarked() { + markedTestCase("char filter", getTokenizers()); + } + + /** + * Fails if a char filter is marked in the superclass with {@link MovedToAnalysisCommon} but + * hasn't been marked in this class with its proper factory. + */ + public void testAllCharFiltersMarked() { + markedTestCase("char filter", getCharFilters()); + } + + /** + * Fails if a char filter is marked in the superclass with {@link MovedToAnalysisCommon} but + * hasn't been marked in this class with its proper factory. + */ + public void testAllTokenFiltersMarked() { + markedTestCase("token filter", getTokenFilters()); + } + + private void markedTestCase(String name, Map> map) { + List unmarked = map.entrySet().stream() + .filter(e -> e.getValue() == MovedToAnalysisCommon.class) + .map(Map.Entry::getKey) + .sorted() + .collect(toList()); + assertEquals(name + " marked in AnalysisFactoryTestCase as moved to analysis-common " + + "but not mapped here", emptyList(), unmarked); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index 07477a7eb0bbc..ffa7989bf73ff 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -107,7 +107,7 @@ import java.util.regex.Pattern; /** - * Alerts us if new analyzers are added to lucene, so we don't miss them. + * Alerts us if new analysis components are added to Lucene, so we don't miss them. *

* If we don't want to expose one for a specific reason, just map it to Void. * The deprecated ones can be mapped to Deprecated.class. @@ -175,7 +175,7 @@ private static String toCamelCase(String s) { .put("apostrophe", ApostropheFilterFactory.class) .put("arabicnormalization", ArabicNormalizationFilterFactory.class) .put("arabicstem", ArabicStemTokenFilterFactory.class) - .put("asciifolding", Void.class) // TODO remove this when core no longer depends on analysis-common + .put("asciifolding", MovedToAnalysisCommon.class) .put("brazilianstem", BrazilianStemTokenFilterFactory.class) .put("bulgarianstem", StemmerTokenFilterFactory.class) .put("cjkbigram", CJKBigramFilterFactory.class) @@ -250,8 +250,8 @@ private static String toCamelCase(String s) { .put("turkishlowercase", LowerCaseTokenFilterFactory.class) .put("type", KeepTypesFilterFactory.class) .put("uppercase", UpperCaseTokenFilterFactory.class) - .put("worddelimiter", Void.class) // TODO remove this when core no longer depends on analysis-common - .put("worddelimitergraph", Void.class) // TODO remove this when core no longer depends on analysis-common + .put("worddelimiter", MovedToAnalysisCommon.class) + .put("worddelimitergraph", MovedToAnalysisCommon.class) .put("flattengraph", FlattenGraphTokenFilterFactory.class) // TODO: these tokenfilters are not yet exposed: useful? @@ -484,4 +484,11 @@ public void testPreBuiltMultiTermAware() { classesThatShouldNotHaveMultiTermSupport.isEmpty()); } + /** + * Marker class for components that have moved to the analysis-common modules. This will be + * removed when the module is complete and these analysis components aren't available to core. + */ + protected static final class MovedToAnalysisCommon { + private MovedToAnalysisCommon() {} + } } From 42338e961c73764b4119d872463e05dc03dc5f3f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Mar 2017 15:01:27 -0400 Subject: [PATCH 18/28] Test cleanup --- .../index/analysis/AnalysisRegistryTests.java | 5 ++--- .../analysis/common/QueryStringWithAnalyzersTests.java | 10 ---------- .../org/elasticsearch/AnalysisFactoryTestCase.java | 1 + 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 403035550f7cb..d4720373ec002 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -177,11 +177,10 @@ public TokenStream create(TokenStream tokenStream) { return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET); } } + @Override public Map> getTokenFilters() { - Map> filters = new HashMap<>(); - filters.put("mock", MockFactory::new); - return filters; + return singletonMap("mock", MockFactory::new); } }; IndexAnalyzers indexAnalyzers = new AnalysisModule(new Environment(settings), singletonList(plugin)).getAnalysisRegistry() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java index 47b58b867e336..7dd53a04494d8 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java @@ -37,16 +37,6 @@ protected Collection> nodePlugins() { return Arrays.asList(CommonAnalysisPlugin.class); } - @Override - protected int maximumNumberOfShards() { - return 7; - } - - @Override - protected int maximumNumberOfReplicas() { - return Math.min(2, cluster().numDataNodes() - 1); - } - /** * Validates that we properly split fields using the word delimiter filter in query_string. */ diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java index ffa7989bf73ff..31c156713e688 100644 --- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java @@ -396,6 +396,7 @@ public void testMultiTermAware() { } } expected.remove(Void.class); + expected.remove(MovedToAnalysisCommon.class); expected.remove(Deprecated.class); Collection> actual = new HashSet<>(); From 7301a6af677d33e495313f0f908797ad66a7cd62 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 14:05:08 -0400 Subject: [PATCH 19/28] Move analyze action implementation to module The analysis-common module will have all the analyzers in it anyway so the only way we're going to get a good test of the analyze action is in that module. --- .../resources/checkstyle_suppressions.xml | 5 - .../elasticsearch/action/ActionModule.java | 5 - .../indices/analyze/AnalyzeResponse.java | 5 +- .../action/IndicesRequestIT.java | 251 +------------ .../indices/TransportAnalyzeActionTests.java | 280 --------------- .../TransportReplicationActionTests.java | 14 - .../elasticsearch/cluster/NoMasterNodeIT.java | 9 - .../analysis/common/CommonAnalysisPlugin.java | 32 +- .../analysis/common}/RestAnalyzeAction.java | 45 ++- .../common}/TransportAnalyzeAction.java | 270 +++++++++----- .../analysis/common/AnalyzeActionTests.java | 336 ++++++++++++------ .../analysis/common/AnalyzeRequestTests.java | 55 +++ .../common}/RestAnalyzeActionTests.java | 37 +- .../common/TransportAnalyzeActionTests.java | 2 - .../action/IndicesRequestTestCase.java | 276 ++++++++++++++ 15 files changed, 827 insertions(+), 795 deletions(-) delete mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java rename {core/src/main/java/org/elasticsearch/rest/action/admin/indices => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/RestAnalyzeAction.java (83%) rename {core/src/main/java/org/elasticsearch/action/admin/indices/analyze => modules/analysis-common/src/main/java/org/elasticsearch/analysis/common}/TransportAnalyzeAction.java (70%) rename core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java (80%) create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java rename {core/src/test/java/org/elasticsearch/rest/action/admin/indices => modules/analysis-common/src/test/java/org/elasticsearch/analysis/common}/RestAnalyzeActionTests.java (88%) create mode 100644 test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index f81d8d32946a3..1d052b519e92a 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -245,7 +245,6 @@ - @@ -1631,7 +1630,6 @@ - @@ -2258,7 +2256,6 @@ - @@ -2931,7 +2928,6 @@ - @@ -3014,7 +3010,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index c1d0541d4ce10..08a22ba2c44f9 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -79,8 +79,6 @@ import org.elasticsearch.action.admin.indices.alias.exists.TransportAliasesExistAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.TransportGetAliasesAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.close.CloseIndexAction; @@ -238,7 +236,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestRestoreSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestSnapshotsStatusAction; import org.elasticsearch.rest.action.admin.cluster.RestVerifyRepositoryAction; -import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction; import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction; import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; @@ -442,7 +439,6 @@ public void reg actions.register(PutMappingAction.INSTANCE, TransportPutMappingAction.class); actions.register(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class); actions.register(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class); - actions.register(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class); actions.register(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class); actions.register(GetIndexTemplatesAction.INSTANCE, TransportGetIndexTemplatesAction.class); actions.register(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class); @@ -549,7 +545,6 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestUpdateSettingsAction(settings, restController)); registerHandler.accept(new RestGetSettingsAction(settings, restController, indexScopedSettings, settingsFilter)); - registerHandler.accept(new RestAnalyzeAction(settings, restController)); registerHandler.accept(new RestGetIndexTemplateAction(settings, restController)); registerHandler.accept(new RestPutIndexTemplateAction(settings, restController)); registerHandler.accept(new RestDeleteIndexTemplateAction(settings, restController)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index e4c6b34d9c7d7..a38b1b4b43145 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -151,7 +151,10 @@ public void writeTo(StreamOutput out) throws IOException { private List tokens; - AnalyzeResponse() { + /** + * For deserialization. + */ + public AnalyzeResponse() { } public AnalyzeResponse(List tokens, DetailAnalyzeResponse detail) { diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java index 0f3812c0cd6ba..51d97bc01cb30 100644 --- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java @@ -19,9 +19,6 @@ package org.elasticsearch.action; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexAction; @@ -66,8 +63,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.action.support.replication.TransportReplicationActionTests; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -77,51 +74,29 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; -import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.transport.TransportChannel; -import org.elasticsearch.transport.TransportInterceptor; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; -import org.junit.After; -import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.emptyIterable; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItem; @ClusterScope(scope = Scope.SUITE, numClientNodes = 1, minNumDataNodes = 2) -public class IndicesRequestIT extends ESIntegTestCase { - - private final List indices = new ArrayList<>(); - +public class IndicesRequestIT extends IndicesRequestTestCase { @Override protected int minimumNumberOfShards() { //makes sure that a reduce is always needed when searching @@ -146,7 +121,10 @@ protected Settings nodeSettings(int ordinal) { @Override protected Collection> nodePlugins() { - return Arrays.asList(InterceptingTransportService.TestPlugin.class, CustomScriptPlugin.class); + List> plugins = new ArrayList<>(); + plugins.addAll(super.nodePlugins()); + plugins.add(CustomScriptPlugin.class); + return plugins; } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -158,24 +136,6 @@ protected Map, Object>> pluginScripts() { } } - @Before - public void setup() { - int numIndices = iterations(1, 5); - for (int i = 0; i < numIndices; i++) { - indices.add("test" + i); - } - for (String index : indices) { - assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias"))); - } - ensureGreen(); - } - - @After - public void cleanUp() { - assertAllRequestsHaveBeenConsumed(); - indices.clear(); - } - public void testGetFieldMappings() { String getFieldMappingsShardAction = GetFieldMappingsAction.NAME + "[index][s]"; interceptTransportActions(getFieldMappingsShardAction); @@ -188,18 +148,6 @@ public void testGetFieldMappings() { assertSameIndices(getFieldMappingsRequest, getFieldMappingsShardAction); } - public void testAnalyze() { - String analyzeShardAction = AnalyzeAction.NAME + "[s]"; - interceptTransportActions(analyzeShardAction); - - AnalyzeRequest analyzeRequest = new AnalyzeRequest(randomIndexOrAlias()); - analyzeRequest.text("text"); - internalCluster().coordOnlyNodeClient().admin().indices().analyze(analyzeRequest).actionGet(); - - clearInterceptedActions(); - assertSameIndices(analyzeRequest, analyzeShardAction); - } - public void testIndex() { String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"}; interceptTransportActions(indexShardActions); @@ -579,191 +527,4 @@ public void testSearchDfsQueryThenFetch() throws Exception { //free context messages are not necessarily sent, but if they are, check their indices assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME); } - - private static void assertSameIndices(IndicesRequest originalRequest, String... actions) { - assertSameIndices(originalRequest, false, actions); - } - - private static void assertSameIndicesOptionalRequests(IndicesRequest originalRequest, String... actions) { - assertSameIndices(originalRequest, true, actions); - } - - private static void assertSameIndices(IndicesRequest originalRequest, boolean optional, String... actions) { - for (String action : actions) { - List requests = consumeTransportRequests(action); - if (!optional) { - assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); - } - for (TransportRequest internalRequest : requests) { - IndicesRequest indicesRequest = convertRequest(internalRequest); - assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), equalTo(originalRequest.indices())); - assertThat(indicesRequest.indicesOptions(), equalTo(originalRequest.indicesOptions())); - } - } - } - private static void assertIndicesSubset(List indices, String... actions) { - //indices returned by each bulk shard request need to be a subset of the original indices - for (String action : actions) { - List requests = consumeTransportRequests(action); - assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); - for (TransportRequest internalRequest : requests) { - IndicesRequest indicesRequest = convertRequest(internalRequest); - for (String index : indicesRequest.indices()) { - assertThat(indices, hasItem(index)); - } - } - } - } - - static IndicesRequest convertRequest(TransportRequest request) { - final IndicesRequest indicesRequest; - if (request instanceof IndicesRequest) { - indicesRequest = (IndicesRequest) request; - } else { - indicesRequest = TransportReplicationActionTests.resolveRequest(request); - } - return indicesRequest; - } - - private String randomIndexOrAlias() { - String index = randomFrom(indices); - if (randomBoolean()) { - return index + "-alias"; - } else { - return index; - } - } - - private String[] randomIndicesOrAliases() { - int count = randomIntBetween(1, indices.size() * 2); //every index has an alias - String[] indices = new String[count]; - for (int i = 0; i < count; i++) { - indices[i] = randomIndexOrAlias(); - } - return indices; - } - - private String[] randomUniqueIndicesOrAliases() { - Set uniqueIndices = new HashSet<>(); - int count = randomIntBetween(1, this.indices.size()); - while (uniqueIndices.size() < count) { - uniqueIndices.add(randomFrom(this.indices)); - } - String[] indices = new String[count]; - int i = 0; - for (String index : uniqueIndices) { - indices[i++] = randomBoolean() ? index + "-alias" : index; - } - return indices; - } - - private static void assertAllRequestsHaveBeenConsumed() { - Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); - for (PluginsService pluginsService : pluginsServices) { - Set>> entries = - pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() - .instance.requests.entrySet(); - assertThat(entries, emptyIterable()); - - } - } - - private static void clearInterceptedActions() { - Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); - for (PluginsService pluginsService : pluginsServices) { - pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() - .instance.clearInterceptedActions(); - } - } - - private static void interceptTransportActions(String... actions) { - Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); - for (PluginsService pluginsService : pluginsServices) { - pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() - .instance.interceptTransportActions(actions); - } - } - - private static List consumeTransportRequests(String action) { - List requests = new ArrayList<>(); - - Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); - for (PluginsService pluginsService : pluginsServices) { - List transportRequests = pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class) - .stream().findFirst().get().instance.consumeRequests(action); - if (transportRequests != null) { - requests.addAll(transportRequests); - } - } - return requests; - } - - public static class InterceptingTransportService implements TransportInterceptor { - - public static class TestPlugin extends Plugin implements NetworkPlugin { - public final InterceptingTransportService instance = new InterceptingTransportService(); - @Override - public List getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, - ThreadContext threadContext) { - return Collections.singletonList(instance); - } - } - - private final Set actions = new HashSet<>(); - - private final Map> requests = new HashMap<>(); - - @Override - public TransportRequestHandler interceptHandler(String action, String executor, - boolean forceExecution, - TransportRequestHandler actualHandler) { - return new InterceptingRequestHandler<>(action, actualHandler); - } - - synchronized List consumeRequests(String action) { - return requests.remove(action); - } - - synchronized void interceptTransportActions(String... actions) { - Collections.addAll(this.actions, actions); - } - - synchronized void clearInterceptedActions() { - actions.clear(); - } - - - private class InterceptingRequestHandler implements TransportRequestHandler { - - private final TransportRequestHandler requestHandler; - private final String action; - - InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { - this.requestHandler = requestHandler; - this.action = action; - } - - @Override - public void messageReceived(T request, TransportChannel channel, Task task) throws Exception { - synchronized (InterceptingTransportService.this) { - if (actions.contains(action)) { - List requestList = requests.get(action); - if (requestList == null) { - requestList = new ArrayList<>(); - requestList.add(request); - requests.put(action, requestList); - } else { - requestList.add(request); - } - } - } - requestHandler.messageReceived(request, channel, task); - } - - @Override - public void messageReceived(T request, TransportChannel channel) throws Exception { - messageReceived(request, channel, null); - } - } - } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java deleted file mode 100644 index 57a83b2c68081..0000000000000 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices; - -import org.apache.lucene.analysis.MockTokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.elasticsearch.index.analysis.AnalysisRegistry; -import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.mapper.AllFieldMapper; -import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; -import org.elasticsearch.plugins.AnalysisPlugin; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.IndexSettingsModule; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; - -/** - * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the - * {@code common-analysis} module. - */ -public class TransportAnalyzeActionTests extends ESTestCase { - - private IndexAnalyzers indexAnalyzers; - private AnalysisRegistry registry; - private Environment environment; - - @Override - public void setUp() throws Exception { - super.setUp(); - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); - - Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard") - .put("index.analysis.analyzer.custom_analyzer.filter", "mock").build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - environment = new Environment(settings); - AnalysisPlugin plugin = new AnalysisPlugin() { - class MockFactory extends AbstractTokenFilterFactory { - MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { - super(indexSettings, name, settings); - } - - @Override - public TokenStream create(TokenStream tokenStream) { - return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET); - } - } - - @Override - public Map> getTokenFilters() { - return singletonMap("mock", MockFactory::new); - } - }; - registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry(); - indexAnalyzers = registry.build(idxSettings); - } - - public void testNoIndexAnalyzers() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); - request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - - request.analyzer(null); - request.tokenizer("whitespace"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("word_delimiter"); - request.text("the qu1ck brown fox"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment); - tokens = analyze.getTokens(); - assertEquals(6, tokens.size()); - assertEquals("qu", tokens.get(1).getTerm()); - assertEquals("1", tokens.get(2).getTerm()); - assertEquals("ck", tokens.get(3).getTerm()); - - request.analyzer(null); - request.tokenizer("whitespace"); - request.addCharFilter("html_strip"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("word_delimiter"); - request.text("

the qu1ck brown fox

"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment); - tokens = analyze.getTokens(); - assertEquals(6, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals("qu", tokens.get(1).getTerm()); - assertEquals("1", tokens.get(2).getTerm()); - assertEquals("ck", tokens.get(3).getTerm()); - assertEquals("brown", tokens.get(4).getTerm()); - assertEquals("fox", tokens.get(5).getTerm()); - } - - public void testFillsAttributes() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); - request.text("the 1 brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals(0, tokens.get(0).getStartOffset()); - assertEquals(3, tokens.get(0).getEndOffset()); - assertEquals(0, tokens.get(0).getPosition()); - assertEquals("", tokens.get(0).getType()); - - assertEquals("1", tokens.get(1).getTerm()); - assertEquals(4, tokens.get(1).getStartOffset()); - assertEquals(5, tokens.get(1).getEndOffset()); - assertEquals(1, tokens.get(1).getPosition()); - assertEquals("", tokens.get(1).getType()); - - assertEquals("brown", tokens.get(2).getTerm()); - assertEquals(6, tokens.get(2).getStartOffset()); - assertEquals(11, tokens.get(2).getEndOffset()); - assertEquals(2, tokens.get(2).getPosition()); - assertEquals("", tokens.get(2).getType()); - - assertEquals("fox", tokens.get(3).getTerm()); - assertEquals(12, tokens.get(3).getStartOffset()); - assertEquals(15, tokens.get(3).getEndOffset()); - assertEquals(3, tokens.get(3).getPosition()); - assertEquals("", tokens.get(3).getType()); - } - - public void testWithIndexAnalyzers() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.text("the quick brown fox"); - request.analyzer("custom_analyzer"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(3, tokens.size()); - assertEquals("quick", tokens.get(0).getTerm()); - assertEquals("brown", tokens.get(1).getTerm()); - assertEquals("fox", tokens.get(2).getTerm()); - - request.analyzer("standard"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals("quick", tokens.get(1).getTerm()); - assertEquals("brown", tokens.get(2).getTerm()); - assertEquals("fox", tokens.get(3).getTerm()); - - // Switch the analyzer out for just a tokenizer - request.analyzer(null); - request.tokenizer("standard"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals("quick", tokens.get(1).getTerm()); - assertEquals("brown", tokens.get(2).getTerm()); - assertEquals("fox", tokens.get(3).getTerm()); - - // Now try applying our token filter - request.addTokenFilter("mock"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - tokens = analyze.getTokens(); - assertEquals(3, tokens.size()); - assertEquals("quick", tokens.get(0).getTerm()); - assertEquals("brown", tokens.get(1).getTerm()); - assertEquals("fox", tokens.get(2).getTerm()); - } - - public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .analyzer("custom_analyzer") - .text("the qu1ck brown fox-dog"), - AllFieldMapper.NAME, null, null, registry, environment)); - assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]"); - } - - public void testUnknown() throws IOException { - boolean notGlobal = randomBoolean(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .analyzer("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global analyzer [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global tokenizer under [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("whitespace") - .addTokenFilter("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global token filter under [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("whitespace") - .addTokenFilter("lowercase") - .addCharFilter("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global char filter under [foobar]"); - } - } - - public void testNonPreBuildTokenFilter() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.tokenizer("whitespace"); - request.addTokenFilter("min_hash"); - request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); - List tokens = analyze.getTokens(); - int default_hash_count = 1; - int default_bucket_size = 512; - int default_hash_set_size = 1; - assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size()); - - } -} diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 1bd943704707d..3834d44f3ec5c 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -116,20 +116,6 @@ import static org.mockito.Mockito.when; public class TransportReplicationActionTests extends ESTestCase { - - /** - * takes a request that was sent by a {@link TransportReplicationAction} and captured - * and returns the underlying request if it's wrapped or the original (cast to the expected type). - * - * This will throw a {@link ClassCastException} if the request is of the wrong type. - */ - public static R resolveRequest(TransportRequest requestOrWrappedRequest) { - if (requestOrWrappedRequest instanceof TransportReplicationAction.ConcreteShardRequest) { - requestOrWrappedRequest = ((TransportReplicationAction.ConcreteShardRequest)requestOrWrappedRequest).getRequest(); - } - return (R) requestOrWrappedRequest; - } - private static ThreadPool threadPool; private ClusterService clusterService; diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 2d2a4cec7e3b0..210379dad99b4 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -102,15 +102,6 @@ public void run() { ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); - - assertThrows(client().admin().indices().prepareAnalyze("test", "this is a test"), - ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE - ); - - assertThrows(client().admin().indices().prepareAnalyze("no_index", "this is a test"), - ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE - ); - assertThrows(client().prepareSearch("test").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index bfd1bbdcc97b8..da511e4d5326b 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -19,15 +19,31 @@ package org.elasticsearch.analysis.common; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Supplier; -public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { +import static java.util.Collections.singletonList; + +public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ActionPlugin { @Override public Map> getTokenFilters() { Map> filters = new HashMap<>(); @@ -36,4 +52,18 @@ public Map> getTokenFilters() { filters.put("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new); return filters; } + + @Override + public List> getActions() { + return singletonList( + new ActionHandler<>(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class)); + } + + @Override + public List getRestHandlers(Settings settings, RestController restController, + ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return singletonList(new RestAnalyzeAction(settings, restController)); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java similarity index 83% rename from core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java index 52ba58e4c38c2..9c4bfe859fd82 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.rest.action.admin.indices; +package org.elasticsearch.analysis.common; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.client.node.NodeClient; @@ -57,8 +57,8 @@ public RestAnalyzeAction(Settings settings, RestController controller) { } @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) + throws IOException { AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index")); try (XContentParser parser = request.contentOrSourceParamParser()) { @@ -67,7 +67,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC throw new IllegalArgumentException("Failed to parse request body", e); } - return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel)); + return channel -> client.admin().indices().analyze( + analyzeRequest, new RestToXContentListener<>(channel)); } static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest) @@ -80,20 +81,25 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.TEXT.match(currentFieldName) + && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.text(parser.text()); - } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { + } else if (Fields.TEXT.match(currentFieldName) + && token == XContentParser.Token.START_ARRAY) { List texts = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { - throw new IllegalArgumentException(currentFieldName + " array element should only contain text"); + throw new IllegalArgumentException(currentFieldName + + " array element should only contain text"); } texts.add(parser.text()); } analyzeRequest.text(texts.toArray(new String[texts.size()])); - } else if (Fields.ANALYZER.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.ANALYZER.match(currentFieldName) + && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.analyzer(parser.text()); - } else if (Fields.FIELD.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.FIELD.match(currentFieldName) + && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.field(parser.text()); } else if (Fields.TOKENIZER.match(currentFieldName)) { if (token == XContentParser.Token.VALUE_STRING) { @@ -101,7 +107,8 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques } else if (token == XContentParser.Token.START_OBJECT) { analyzeRequest.tokenizer(parser.map()); } else { - throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting"); + throw new IllegalArgumentException(currentFieldName + + " should be tokenizer's name or setting"); } } else if (Fields.TOKEN_FILTERS.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { @@ -123,28 +130,32 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques } else if (token == XContentParser.Token.START_OBJECT) { analyzeRequest.addCharFilter(parser.map()); } else { - throw new IllegalArgumentException(currentFieldName - + " array element should contain char filter's name or setting"); + throw new IllegalArgumentException(currentFieldName + " array element " + + "should contain char filter's name or setting"); } } } else if (Fields.EXPLAIN.match(currentFieldName)) { if (parser.isBooleanValue()) { analyzeRequest.explain(parser.booleanValue()); } else { - throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'"); + throw new IllegalArgumentException(currentFieldName + + " must be either 'true' or 'false'"); } - } else if (Fields.ATTRIBUTES.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { + } else if (Fields.ATTRIBUTES.match(currentFieldName) + && token == XContentParser.Token.START_ARRAY) { List attributes = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { - throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name"); + throw new IllegalArgumentException(currentFieldName + + " array element should only contain attribute name"); } attributes.add(parser.text()); } analyzeRequest.attributes(attributes.toArray(new String[attributes.size()])); } else { - throw new IllegalArgumentException("Unknown parameter [" - + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] "); + throw new IllegalArgumentException("Unknown parameter [" + currentFieldName + + "] in request body or parameter is of the wrong type[" + + token + "] "); } } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java similarity index 70% rename from core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java rename to modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java index d7e299b1cf1b5..41b899981fb84 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.action.admin.indices.analyze; +package org.elasticsearch.analysis.common; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -30,6 +30,10 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.DetailAnalyzeResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; @@ -71,19 +75,28 @@ import java.util.Set; import java.util.TreeMap; +import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER; +import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_FILTER; +import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER; +import static org.elasticsearch.index.analysis.AnalysisRegistry.getSettingsFromIndexSettings; + /** - * Transport action used to execute analyze requests + * Transport action used to execute analyze requests. */ -public class TransportAnalyzeAction extends TransportSingleShardAction { +public class TransportAnalyzeAction extends + TransportSingleShardAction { private final IndicesService indicesService; private final Environment environment; @Inject - public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, - IndicesService indicesService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) { - super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, AnalyzeRequest::new, ThreadPool.Names.INDEX); + public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, + ClusterService clusterService, TransportService transportService, + IndicesService indicesService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) { + super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, + actionFilters, indexNameExpressionResolver, AnalyzeRequest::new, + ThreadPool.Names.INDEX); this.indicesService = indicesService; this.environment = environment; } @@ -128,7 +141,8 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId Analyzer analyzer = null; if (request.field() != null) { if (indexService == null) { - throw new IllegalArgumentException("No index provided, and trying to analyzer based on a specific field which requires the index parameter"); + throw new IllegalArgumentException("No index provided, and trying to analyzer " + + "based on a specific field which requires the index parameter"); } MappedFieldType fieldType = indexService.mapperService().fullName(request.field()); if (fieldType != null) { @@ -141,7 +155,8 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId analyzer = fieldType.indexAnalyzer(); } } else { - throw new IllegalArgumentException("Can't process field [" + request.field() + "], Analysis requests are only supported on tokenized fields"); + throw new IllegalArgumentException("Can't process field [" + request.field() + + "], Analysis requests are only supported on tokenized fields"); } field = fieldType.name(); } @@ -154,40 +169,48 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId } } final AnalysisRegistry analysisRegistry = indicesService.getAnalysis(); - return analyze(request, field, analyzer, indexService != null ? indexService.getIndexAnalyzers() : null, analysisRegistry, environment); + IndexAnalyzers analzyers = + indexService == null ? null : indexService.getIndexAnalyzers(); + return analyze(request, field, analyzer, analzyers, analysisRegistry, environment); } catch (IOException e) { throw new ElasticsearchException("analysis failed", e); } - } - public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, IndexAnalyzers indexAnalyzers, AnalysisRegistry analysisRegistry, Environment environment) throws IOException { - + public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, + IndexAnalyzers indexAnalyzers, AnalysisRegistry analysisRegistry, + Environment environment) throws IOException { boolean closeAnalyzer = false; if (analyzer == null && request.analyzer() != null) { if (indexAnalyzers == null) { analyzer = analysisRegistry.getAnalyzer(request.analyzer()); if (analyzer == null) { - throw new IllegalArgumentException("failed to find global analyzer [" + request.analyzer() + "]"); + throw new IllegalArgumentException("failed to find global analyzer [" + + request.analyzer() + "]"); } } else { analyzer = indexAnalyzers.get(request.analyzer()); if (analyzer == null) { - throw new IllegalArgumentException("failed to find analyzer [" + request.analyzer() + "]"); + throw new IllegalArgumentException("failed to find analyzer [" + + request.analyzer() + "]"); } } - } else if (request.tokenizer() != null) { - final IndexSettings indexSettings = indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings(); - TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers, analysisRegistry, environment); + final IndexSettings indexSettings = + indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings(); + TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers, + analysisRegistry, environment); TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0]; - tokenFilterFactories = getTokenFilterFactories(request, indexSettings, analysisRegistry, environment, tokenFilterFactories); + tokenFilterFactories = getTokenFilterFactories(request, indexSettings, analysisRegistry, + environment, tokenFilterFactories); CharFilterFactory[] charFilterFactories = new CharFilterFactory[0]; - charFilterFactories = getCharFilterFactories(request, indexSettings, analysisRegistry, environment, charFilterFactories); + charFilterFactories = getCharFilterFactories(request, indexSettings, analysisRegistry, + environment, charFilterFactories); - analyzer = new CustomAnalyzer(tokenizerFactory, charFilterFactories, tokenFilterFactories); + analyzer = new CustomAnalyzer( + tokenizerFactory, charFilterFactories, tokenFilterFactories); closeAnalyzer = true; } else if (analyzer == null) { if (indexAnalyzers == null) { @@ -216,7 +239,8 @@ public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Anal return new AnalyzeResponse(tokens, detail); } - private static List simpleAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) { + private static List simpleAnalyze(AnalyzeRequest request, + Analyzer analyzer, String field) { List tokens = new ArrayList<>(); int lastPosition = -1; int lastOffset = 0; @@ -224,7 +248,8 @@ private static List simpleAnalyze(AnalyzeRequest r try (TokenStream stream = analyzer.tokenStream(field, text)) { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); - PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); + PositionIncrementAttribute posIncr = stream.addAttribute( + PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); @@ -234,9 +259,9 @@ private static List simpleAnalyze(AnalyzeRequest r if (increment > 0) { lastPosition = lastPosition + increment; } - tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), - lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null)); - + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, + lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), + posLen.getPositionLength(), type.type(), null)); } stream.end(); lastOffset += offset.endOffset(); @@ -251,7 +276,8 @@ private static List simpleAnalyze(AnalyzeRequest r return tokens; } - private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) { + private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, + String field) { DetailAnalyzeResponse detailResponse; final Set includeAttributes = new HashSet<>(); if (request.attributes() != null) { @@ -263,7 +289,8 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy CustomAnalyzer customAnalyzer = null; if (analyzer instanceof CustomAnalyzer) { customAnalyzer = (CustomAnalyzer) analyzer; - } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { + } else if (analyzer instanceof NamedAnalyzer && + ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer(); } @@ -273,8 +300,11 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory(); TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters(); - String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length]; - TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? tokenFilterFactories.length : 0]; + int charFilterLength = charFilterFactories == null ? 0 : charFilterFactories.length; + String[][] charFiltersTexts = new String[charFilterLength][request.text().length]; + int tokenFilterLength = tokenFilterFactories == null ? 0 : tokenFilterFactories.length; + TokenListCreator[] tokenFiltersTokenListCreator = + new TokenListCreator[tokenFilterLength]; TokenListCreator tokenizerTokenListCreator = new TokenListCreator(); @@ -284,10 +314,12 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy Reader reader = new FastStringReader(charFilteredSource); if (charFilterFactories != null) { - for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; charFilterIndex++) { + for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; + charFilterIndex++) { reader = charFilterFactories[charFilterIndex].create(reader); Reader readerForWriteOut = new FastStringReader(charFilteredSource); - readerForWriteOut = charFilterFactories[charFilterIndex].create(readerForWriteOut); + readerForWriteOut = charFilterFactories[charFilterIndex] + .create(readerForWriteOut); charFilteredSource = writeCharStream(readerForWriteOut); charFiltersTexts[charFilterIndex][textIndex] = charFilteredSource; } @@ -296,36 +328,49 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy // analyzing only tokenizer Tokenizer tokenizer = tokenizerFactory.create(); tokenizer.setReader(reader); - tokenizerTokenListCreator.analyze(tokenizer, customAnalyzer, field, includeAttributes); + tokenizerTokenListCreator.analyze( + tokenizer, customAnalyzer, field, includeAttributes); // analyzing each tokenfilter if (tokenFilterFactories != null) { - for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; tokenFilterIndex++) { + for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; + tokenFilterIndex++) { if (tokenFiltersTokenListCreator[tokenFilterIndex] == null) { tokenFiltersTokenListCreator[tokenFilterIndex] = new TokenListCreator(); } TokenStream stream = createStackedTokenStream(request.text()[textIndex], - charFilterFactories, tokenizerFactory, tokenFilterFactories, tokenFilterIndex + 1); - tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, customAnalyzer, field, includeAttributes); + charFilterFactories, tokenizerFactory, + tokenFilterFactories, tokenFilterIndex + 1); + tokenFiltersTokenListCreator[tokenFilterIndex].analyze( + stream, customAnalyzer, field, includeAttributes); } } } - DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length]; + DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = + new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length]; if (charFilterFactories != null) { - for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; charFilterIndex++) { + for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; + charFilterIndex++) { charFilteredLists[charFilterIndex] = new DetailAnalyzeResponse.CharFilteredText( - charFilterFactories[charFilterIndex].name(), charFiltersTexts[charFilterIndex]); + charFilterFactories[charFilterIndex].name(), + charFiltersTexts[charFilterIndex]); } } - DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length]; + DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = + new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length]; if (tokenFilterFactories != null) { - for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFiltersTokenListCreator.length; tokenFilterIndex++) { + for (int tokenFilterIndex = 0; + tokenFilterIndex < tokenFiltersTokenListCreator.length; + tokenFilterIndex++) { tokenFilterLists[tokenFilterIndex] = new DetailAnalyzeResponse.AnalyzeTokenList( - tokenFilterFactories[tokenFilterIndex].name(), tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens()); + tokenFilterFactories[tokenFilterIndex].name(), + tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens()); } } - detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); + detailResponse = new DetailAnalyzeResponse(charFilteredLists, + new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), + tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); } else { String name; if (analyzer instanceof NamedAnalyzer) { @@ -339,12 +384,15 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy tokenListCreator.analyze(analyzer.tokenStream(field, text), analyzer, field, includeAttributes); } - detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList(name, tokenListCreator.getArrayTokens())); + detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList( + name, tokenListCreator.getArrayTokens())); } return detailResponse; } - private static TokenStream createStackedTokenStream(String source, CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, TokenFilterFactory[] tokenFilterFactories, int current) { + private static TokenStream createStackedTokenStream(String source, + CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, + TokenFilterFactory[] tokenFilterFactories, int current) { Reader reader = new FastStringReader(source); for (CharFilterFactory charFilterFactory : charFilterFactories) { reader = charFilterFactory.create(reader); @@ -385,11 +433,13 @@ private static class TokenListCreator { tokens = new ArrayList<>(); } - private void analyze(TokenStream stream, Analyzer analyzer, String field, Set includeAttributes) { + private void analyze(TokenStream stream, Analyzer analyzer, + String field, Set includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); - PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); + PositionIncrementAttribute posIncr = + stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); @@ -399,8 +449,10 @@ private void analyze(TokenStream stream, Analyzer analyzer, String field, Set 0) { lastPosition = lastPosition + increment; } - tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), - lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, + lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), + posLen.getPositionLength(), type.type(), + extractExtendedAttributes(stream, includeAttributes))); } stream.end(); @@ -431,7 +483,8 @@ private AnalyzeResponse.AnalyzeToken[] getArrayTokens() { * @param includeAttributes filtering attributes * @return Map<key value> */ - private static Map extractExtendedAttributes(TokenStream stream, final Set includeAttributes) { + private static Map extractExtendedAttributes(TokenStream stream, + final Set includeAttributes) { final Map extendedAttributes = new TreeMap<>(); stream.reflectWith((attClass, key, value) -> { @@ -447,7 +500,9 @@ private static Map extractExtendedAttributes(TokenStream stream, if (TypeAttribute.class.isAssignableFrom(attClass)) { return; } - if (includeAttributes == null || includeAttributes.isEmpty() || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { + if (includeAttributes == null + || includeAttributes.isEmpty() + || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { if (value instanceof BytesRef) { final BytesRef p = (BytesRef) value; value = p.toString(); @@ -459,8 +514,9 @@ private static Map extractExtendedAttributes(TokenStream stream, return extendedAttributes; } - private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry, - Environment environment, CharFilterFactory[] charFilterFactories) throws IOException { + private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, + IndexSettings indexSettings, AnalysisRegistry analysisRegistry, Environment environment, + CharFilterFactory[] charFilterFactories) throws IOException { if (request.charFilters() != null && request.charFilters().size() > 0) { charFilterFactories = new CharFilterFactory[request.charFilters().size()]; for (int i = 0; i < request.charFilters().size(); i++) { @@ -470,43 +526,55 @@ private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request Settings settings = getAnonymousSettings(charFilter.definition); String charFilterTypeName = settings.get("type"); if (charFilterTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous char filter: " + charFilter.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous " + + "char filter: " + charFilter.definition); } AnalysisModule.AnalysisProvider charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilterTypeName); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global char filter under [" + charFilterTypeName + "]"); + throw new IllegalArgumentException("failed to find global char filter " + + "under [" + charFilterTypeName + "]"); } // Need to set anonymous "name" of char_filter - charFilterFactories[i] = charFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_charfilter_[" + i + "]", settings); + charFilterFactories[i] = charFilterFactoryFactory.get( + getNaIndexSettings(settings), environment, + "_anonymous_charfilter_[" + i + "]", settings); } else { AnalysisModule.AnalysisProvider charFilterFactoryFactory; if (indexSettings == null) { - charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name); + charFilterFactoryFactory = + analysisRegistry.getCharFilterProvider(charFilter.name); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global char filter under [" + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find global char filter" + + "under [" + charFilter.name + "]"); } - charFilterFactories[i] = charFilterFactoryFactory.get(environment, charFilter.name); + charFilterFactories[i] = charFilterFactoryFactory.get( + environment, charFilter.name); } else { - charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, indexSettings); + charFilterFactoryFactory = analysisRegistry.getCharFilterProvider( + charFilter.name, indexSettings); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + + charFilter.name + "]"); } - charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, environment, charFilter.name, - AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, - AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name)); + Settings analysisSettings = getSettingsFromIndexSettings( + indexSettings, INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name); + charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, + environment, charFilter.name, analysisSettings); } } if (charFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + + charFilter.name + "]"); } } } return charFilterFactories; } - private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry, - Environment environment, TokenFilterFactory[] tokenFilterFactories) throws IOException { + private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, + IndexSettings indexSettings, AnalysisRegistry analysisRegistry, Environment environment, + TokenFilterFactory[] tokenFilterFactories) throws IOException { if (request.tokenFilters() != null && request.tokenFilters().size() > 0) { tokenFilterFactories = new TokenFilterFactory[request.tokenFilters().size()]; for (int i = 0; i < request.tokenFilters().size(); i++) { @@ -516,43 +584,55 @@ private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest reque Settings settings = getAnonymousSettings(tokenFilter.definition); String filterTypeName = settings.get("type"); if (filterTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous token filter: " + tokenFilter.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous " + + "token filter: " + tokenFilter.definition); } AnalysisModule.AnalysisProvider tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(filterTypeName); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global token filter under [" + filterTypeName + "]"); + throw new IllegalArgumentException("failed to find global token filter " + + "under [" + filterTypeName + "]"); } // Need to set anonymous "name" of tokenfilter - tokenFilterFactories[i] = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter_[" + i + "]", settings); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get( + getNaIndexSettings(settings), environment, + "_anonymous_tokenfilter_[" + i + "]", settings); } else { AnalysisModule.AnalysisProvider tokenFilterFactoryFactory; if (indexSettings == null) { - tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name); + tokenFilterFactoryFactory = + analysisRegistry.getTokenFilterProvider(tokenFilter.name); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global token filter under [" + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find global token filter " + + "under [" + tokenFilter.name + "]"); } - tokenFilterFactories[i] = tokenFilterFactoryFactory.get(environment, tokenFilter.name); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get( + environment, tokenFilter.name); } else { - tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name, indexSettings); + tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider( + tokenFilter.name, indexSettings); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find token filter under [" + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find token filter under [" + + tokenFilter.name + "]"); } - tokenFilterFactories[i] = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name, - AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, - AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name)); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get( + indexSettings, environment, tokenFilter.name, + getSettingsFromIndexSettings(indexSettings, + INDEX_ANALYSIS_FILTER + "." + tokenFilter.name)); } } if (tokenFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find or create token filter under [" + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find or create token filter " + + "under [" + tokenFilter.name + "]"); } } } return tokenFilterFactories; } - private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, IndexAnalyzers indexAnalzyers, - AnalysisRegistry analysisRegistry, Environment environment) throws IOException { + private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, + IndexAnalyzers indexAnalzyers, AnalysisRegistry analysisRegistry, + Environment environment) throws IOException { TokenizerFactory tokenizerFactory; final AnalyzeRequest.NameOrDefinition tokenizer = request.tokenizer(); // parse anonymous settings @@ -560,38 +640,46 @@ private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, In Settings settings = getAnonymousSettings(tokenizer.definition); String tokenizerTypeName = settings.get("type"); if (tokenizerTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous tokenizer: " + tokenizer.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous " + + "tokenizer: " + tokenizer.definition); } AnalysisModule.AnalysisProvider tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizerTypeName); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizerTypeName + "]"); + throw new IllegalArgumentException("failed to find global tokenizer under [" + + tokenizerTypeName + "]"); } // Need to set anonymous "name" of tokenizer - tokenizerFactory = tokenizerFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings); + tokenizerFactory = tokenizerFactoryFactory.get( + getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings); } else { AnalysisModule.AnalysisProvider tokenizerFactoryFactory; if (indexAnalzyers == null) { tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizer.name + "]"); + throw new IllegalArgumentException("failed to find global tokenizer under [" + + tokenizer.name + "]"); } tokenizerFactory = tokenizerFactoryFactory.get(environment, tokenizer.name); } else { - tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name, indexAnalzyers.getIndexSettings()); + tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider( + tokenizer.name, indexAnalzyers.getIndexSettings()); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find tokenizer under [" + tokenizer.name + "]"); + throw new IllegalArgumentException("failed to find tokenizer under [" + + tokenizer.name + "]"); } - tokenizerFactory = tokenizerFactoryFactory.get(indexAnalzyers.getIndexSettings(), environment, tokenizer.name, - AnalysisRegistry.getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(), - AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name)); + tokenizerFactory = tokenizerFactoryFactory.get( + indexAnalzyers.getIndexSettings(), environment, tokenizer.name, + getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(), + INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name)); } } return tokenizerFactory; } private static IndexSettings getNaIndexSettings(Settings settings) { - IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build(); + IndexMetaData metaData = IndexMetaData + .builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build(); return new IndexSettings(metaData, Settings.EMPTY); } diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java similarity index 80% rename from core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java index 4f9b13d91cee6..7626334680893 100644 --- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.indices.analyze; +package org.elasticsearch.analysis.common; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; @@ -37,13 +37,14 @@ import static org.hamcrest.Matchers.startsWith; -public class AnalyzeActionIT extends ESIntegTestCase { +public class AnalyzeActionTests extends ESIntegTestCase { public void testSimpleAnalyzerTests() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); for (int i = 0; i < 10; i++) { - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "this is a test").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze(indexOrAlias(), "this is a test").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("this")); @@ -73,25 +74,32 @@ public void testSimpleAnalyzerTests() throws Exception { } public void testAnalyzeNumericField() throws IOException { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "long", "type=long", "double", "type=double")); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .addMapping("test", "long", "type=long", "double", "type=double")); ensureGreen("test"); - expectThrows(IllegalArgumentException.class, - () -> client().admin().indices().prepareAnalyze(indexOrAlias(), "123").setField("long").get()); + expectThrows(IllegalArgumentException.class, () -> client().admin().indices() + .prepareAnalyze(indexOrAlias(), "123").setField("long").get()); - expectThrows(IllegalArgumentException.class, - () -> client().admin().indices().prepareAnalyze(indexOrAlias(), "123.0").setField("double").get()); + expectThrows(IllegalArgumentException.class, () -> client().admin().indices() + .prepareAnalyze(indexOrAlias(), "123.0").setField("double").get()); } public void testAnalyzeWithNoIndex() throws Exception { - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setTokenizer("keyword") + .addTokenFilter("lowercase").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("standard").addTokenFilter("lowercase").addTokenFilter("reverse").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setTokenizer("standard") + .addTokenFilter("lowercase") + .addTokenFilter("reverse").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("siht")); @@ -102,7 +110,9 @@ public void testAnalyzeWithNoIndex() throws Exception { token = analyzeResponse.getTokens().get(3); assertThat(token.getTerm(), equalTo("tset")); - analyzeResponse = client().admin().indices().prepareAnalyze("of course").setTokenizer("standard").addTokenFilter("stop").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("of course") + .setTokenizer("standard") + .addTokenFilter("stop").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("course")); assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1)); @@ -115,23 +125,40 @@ public void testAnalyzeWithCharFilters() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.builder().put(indexSettings()) .put("index.analysis.char_filter.custom_mapping.type", "mapping") - .putArray("index.analysis.char_filter.custom_mapping.mappings", "ph=>f", "qu=>q") - .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard") - .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "custom_mapping"))); + .putArray("index.analysis.char_filter.custom_mapping.mappings", + "ph=>f", "qu=>q") + .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", + "standard") + .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", + "custom_mapping"))); ensureGreen(); - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("

THIS IS A

TEST").setTokenizer("standard").addCharFilter("html_strip").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("

THIS IS A

TEST") + .setTokenizer("standard") + .addCharFilter("html_strip").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("html_strip").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setTokenizer("keyword") + .addTokenFilter("lowercase") + .addCharFilter("html_strip").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); - analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "jeff quit phish").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("custom_mapping").get(); + analyzeResponse = client().admin().indices() + .prepareAnalyze(indexOrAlias(), "jeff quit phish") + .setTokenizer("keyword") + .addTokenFilter("lowercase") + .addCharFilter("custom_mapping").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("jeff qit fish")); - analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "jeff quit fish").setTokenizer("standard").addCharFilter("html_strip").addCharFilter("custom_mapping").get(); + analyzeResponse = client().admin().indices() + .prepareAnalyze(indexOrAlias(), "jeff quit fish") + .setTokenizer("standard") + .addCharFilter("html_strip") + .addCharFilter("custom_mapping").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(3)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("jeff")); @@ -150,7 +177,10 @@ public void testAnalyzeWithNonDefaultPostionLength() throws Exception { .putArray("index.analysis.analyzer.custom_syns.filter", "lowercase", "syns"))); ensureGreen(); - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("say what the fudge").setIndex("test").setAnalyzer("custom_syns").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("say what the fudge") + .setIndex("test") + .setAnalyzer("custom_syns").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(5)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); @@ -197,7 +227,8 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception { .setType("document").setSource("simple", "type=text,analyzer=simple").get(); for (int i = 0; i < 10; i++) { - final AnalyzeRequestBuilder requestBuilder = client().admin().indices().prepareAnalyze("THIS IS A TEST"); + final AnalyzeRequestBuilder requestBuilder = client().admin().indices() + .prepareAnalyze("THIS IS A TEST"); requestBuilder.setIndex(indexOrAlias()); requestBuilder.setField("document.simple"); AnalyzeResponse analyzeResponse = requestBuilder.get(); @@ -212,10 +243,12 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception { // issue #5974 public void testThatStandardAndDefaultAnalyzersAreSame() throws Exception { - AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("standard").get(); + AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test") + .setAnalyzer("standard").get(); assertTokens(response, "this", "is", "a", "test"); - response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("default").get(); + response = client().admin().indices().prepareAnalyze("this is a test") + .setAnalyzer("default").get(); assertTokens(response, "this", "is", "a", "test"); response = client().admin().indices().prepareAnalyze("this is a test").get(); @@ -237,8 +270,8 @@ public void testAnalyzerWithMultiValues() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); - client().admin().indices().preparePutMapping("test") - .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + client().admin().indices().preparePutMapping("test").setType("document") + .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"}; @@ -277,36 +310,50 @@ public void testDetailAnalyze() throws Exception { ensureGreen(); for (int i = 0; i < 10; i++) { - AnalyzeResponse analyzeResponse = admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText("THIS IS A PHISH") - .setExplain(true).addCharFilter("my_mapping").setTokenizer("keyword").addTokenFilter("lowercase").get(); + AnalyzeResponse analyzeResponse = admin().indices().prepareAnalyze() + .setIndex(indexOrAlias()).setText("THIS IS A PHISH") + .setExplain(true) + .addCharFilter("my_mapping") + .setTokenizer("keyword") + .addTokenFilter("lowercase").get(); assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("THIS IS A FISH")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], + equalTo("THIS IS A FISH")); //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("THIS IS A FISH")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), + equalTo("THIS IS A FISH")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), + equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), + equalTo(15)); //tokenfilters assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("this is a fish")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(15)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), + equalTo("this is a fish")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), + equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), + equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), + equalTo(15)); } } public void testDetailAnalyzeWithNoIndex() throws Exception { //analyzer only - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setExplain(true).setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("THIS IS A TEST") + .setExplain(true) + .setAnalyzer("simple").get(); assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); @@ -317,8 +364,10 @@ public void testDetailAnalyzeWithNoIndex() throws Exception { public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { //analyzer only - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setExplain(true).setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("THIS IS A TEST") + .setExplain(true) + .setAnalyzer("simple").get(); assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); @@ -328,23 +377,28 @@ public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { //custom analyzer analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setExplain(true).addCharFilter("html_strip").setTokenizer("keyword").addTokenFilter("lowercase").get(); + .setExplain(true) + .addCharFilter("html_strip") + .setTokenizer("keyword") + .addTokenFilter("lowercase").get(); assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("html_strip")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("\nTHIS IS A TEST\n")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], + equalTo("\nTHIS IS A TEST\n")); //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("\nTHIS IS A TEST\n")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), + equalTo("\nTHIS IS A TEST\n")); //tokenfilters assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("\nthis is a test\n")); - + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), + equalTo("\nthis is a test\n")); //check other attributes analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") @@ -353,35 +407,44 @@ public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), + equalTo("troubl")); String[] expectedAttributesKey = { "bytes", "positionLength", "keyword"}; - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), + equalTo(expectedAttributesKey.length)); Object extendedAttribute; for (String key : expectedAttributesKey) { - extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2] + .getAttributes().get(key); assertThat(extendedAttribute, notNullValue()); } } public void testDetailAnalyzeSpecifyAttributes() throws Exception { - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") - .setExplain(true).setTokenizer("standard").addTokenFilter("snowball").setAttributes("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze("This is troubled") + .setExplain(true) + .setTokenizer("standard") + .addTokenFilter("snowball") + .setAttributes("keyword").get(); assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); - String[] expectedAttributesKey = { - "keyword"}; - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), + equalTo("troubl")); + String[] expectedAttributesKey = {"keyword"}; + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), + equalTo(expectedAttributesKey.length)); Object extendedAttribute; for (String key : expectedAttributesKey) { - extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2] + .getAttributes().get(key); assertThat(extendedAttribute, notNullValue()); } } @@ -390,11 +453,17 @@ public void testDetailAnalyzeWithMultiValues() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); client().admin().indices().preparePutMapping("test") - .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + .setType("document") + .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"}; - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) - .setExplain(true).setField("simple").setText(texts).execute().get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze() + .setIndex(indexOrAlias()) + .setText(texts) + .setExplain(true) + .setField("simple") + .setText(texts).get(); assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple")); assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7)); @@ -424,23 +493,30 @@ public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exceptio .put("index.analysis.analyzer.test_analyzer.position_increment_gap", "100") .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard") .putArray("index.analysis.analyzer.test_analyzer.char_filter", "my_mapping") - .putArray("index.analysis.analyzer.test_analyzer.filter", "snowball", "lowercase"))); + .putArray("index.analysis.analyzer.test_analyzer.filter", + "snowball", "lowercase"))); ensureGreen(); client().admin().indices().preparePutMapping("test") - .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + .setType("document") + .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); //only analyzer = String[] texts = new String[]{"this is a PHISH", "the troubled text"}; - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) - .setExplain(true).setAnalyzer("test_analyzer").setText(texts).execute().get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze() + .setIndex(indexOrAlias()) + .setText(texts) + .setExplain(true) + .setAnalyzer("test_analyzer").get(); // charfilter assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(2)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("this is a FISH")); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[1], equalTo("the troubled text")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], + equalTo("this is a FISH")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[1], + equalTo("the troubled text")); // tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("standard")); @@ -525,54 +601,81 @@ public void testCustomTokenFilterInRequest() throws Exception { assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("whitespace")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("Foo")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), + equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), + equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("buzz")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(4)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), + equalTo(4)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(8)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), + equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("test")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(9)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), + equalTo(9)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(13)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), + equalTo(1)); // tokenfilter(lowercase) assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("foo")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), equalTo(1)); - - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), equalTo("buzz")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), equalTo(4)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), equalTo(8)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), equalTo(1)); - - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("test")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), equalTo(9)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), equalTo(13)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), + equalTo("foo")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), + equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), + equalTo(3)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), + equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), + equalTo(1)); + + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), + equalTo("buzz")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), + equalTo(4)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), + equalTo(8)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), + equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), + equalTo(1)); + + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), + equalTo("test")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), + equalTo(9)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), + equalTo(13)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), + equalTo(2)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), + equalTo(1)); // tokenfilter({"type": "stop", "stopwords": ["foo", "buzz"]}) - assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("_anonymous_tokenfilter_[1]")); + assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), + equalTo("_anonymous_tokenfilter_[1]")); assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getTerm(), equalTo("test")); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), equalTo(9)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), equalTo(13)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getTerm(), + equalTo("test")); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), + equalTo(9)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), + equalTo(13)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), + equalTo(2)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), + equalTo(1)); } @@ -591,16 +694,21 @@ public void testCustomCharFilterInRequest() throws Exception { assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("_anonymous_charfilter_[0]")); + assertThat(analyzeResponse.detail().charfilters()[0].getName(), + equalTo("_anonymous_charfilter_[0]")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("jeff qit fish")); - //tokenizer + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], + equalTo("jeff qit fish")); + // tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("jeff qit fish")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), + equalTo("jeff qit fish")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), + equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), + equalTo(1)); } @@ -617,33 +725,42 @@ public void testCustomTokenizerInRequest() throws Exception { .setExplain(true) .get(); - //tokenizer + // tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("_anonymous_tokenizer")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("go")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), + equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(2)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), + equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("oo")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), + equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), + equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("od")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(2)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), + equalTo(2)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(4)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), + equalTo(1)); } public void testAnalyzeKeywordField() throws IOException { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "keyword", "type=keyword")); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .addMapping("test", "keyword", "type=keyword")); ensureGreen("test"); - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze(indexOrAlias(), "ABC") + .setField("keyword").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("ABC")); @@ -661,7 +778,9 @@ public void testAnalyzeNormalizedKeywordField() throws IOException { .addMapping("test", "keyword", "type=keyword,normalizer=my_normalizer")); ensureGreen("test"); - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices() + .prepareAnalyze(indexOrAlias(), "ABC") + .setField("keyword").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("abc")); @@ -669,6 +788,5 @@ public void testAnalyzeNormalizedKeywordField() throws IOException { assertThat(token.getEndOffset(), equalTo(3)); assertThat(token.getPosition(), equalTo(0)); assertThat(token.getPositionLength(), equalTo(1)); - } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java new file mode 100644 index 0000000000000..f0b9d48bace60 --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.action.IndicesRequestTestCase; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.plugins.Plugin; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Tests routing the analyze request. + */ +public class AnalyzeRequestTests extends IndicesRequestTestCase { + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(); + plugins.addAll(super.nodePlugins()); + plugins.add(CommonAnalysisPlugin.class); + return plugins; + } + + public void testAnalyze() { + String analyzeShardAction = AnalyzeAction.NAME + "[s]"; + interceptTransportActions(analyzeShardAction); + + AnalyzeRequest analyzeRequest = new AnalyzeRequest(randomIndexOrAlias()); + analyzeRequest.text("text"); + internalCluster().coordOnlyNodeClient().admin().indices() + .analyze(analyzeRequest).actionGet(); + + clearInterceptedActions(); + assertSameIndices(analyzeRequest, analyzeShardAction); + } +} diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java similarity index 88% rename from core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java index 958b9e5222fad..5a0c0dc602e10 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.rest.action.admin.indices; +package org.elasticsearch.analysis.common; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.common.bytes.BytesArray; @@ -91,27 +91,32 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti assertThat(analyzeRequest.charFilters().get(0).definition, notNullValue()); } - public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception { - RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class)); + public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() + throws Exception { + RestAnalyzeAction action = new RestAnalyzeAction( + Settings.EMPTY, mock(RestController.class)); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> action.handleRequest(request, null, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } - public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception { + public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() + throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); XContentParser invalidContent = createParser(XContentFactory.jsonBuilder() .startObject() .field("text", "THIS IS A TEST") .field("unknown", "keyword") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest)); assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]")); } - public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() throws Exception { + public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() + throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); XContentParser invalidExplain = createParser(XContentFactory.jsonBuilder() .startObject() @@ -130,8 +135,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, - new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [filters]")); } @@ -142,8 +147,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("token_filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, - new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [token_filters]")); } @@ -154,8 +159,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("char_filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, - new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [char_filters]")); } @@ -166,8 +171,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("token_filter", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, - new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [token_filter]")); } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java index c7d0abceed3db..acef0c426399f 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -154,7 +153,6 @@ public void testFillsAttributes() throws IOException { } public void testWithIndexAnalyzers() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); request.analyzer("standard"); request.text("the quick brown fox"); diff --git a/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java b/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java new file mode 100644 index 0000000000000..cd43fcaeffe93 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java @@ -0,0 +1,276 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action; + +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.action.support.replication.TransportReplicationAction; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.plugins.NetworkPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportInterceptor; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestHandler; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; + +public class IndicesRequestTestCase extends ESIntegTestCase { + private final List indices = new ArrayList<>(); + + @Override + protected Collection> nodePlugins() { + return singletonList(InterceptingTransportService.TestPlugin.class); + } + + @Before + public void setup() { + int numIndices = iterations(1, 5); + for (int i = 0; i < numIndices; i++) { + indices.add("test" + i); + } + for (String index : indices) { + assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias"))); + } + ensureGreen(); + } + + @After + public void cleanUp() { + assertAllRequestsHaveBeenConsumed(); + indices.clear(); + } + + protected String randomIndexOrAlias() { + String index = randomFrom(indices); + if (randomBoolean()) { + return index + "-alias"; + } else { + return index; + } + } + + protected String[] randomIndicesOrAliases() { + int count = randomIntBetween(1, indices.size() * 2); //every index has an alias + String[] indices = new String[count]; + for (int i = 0; i < count; i++) { + indices[i] = randomIndexOrAlias(); + } + return indices; + } + + protected String[] randomUniqueIndicesOrAliases() { + Set uniqueIndices = new HashSet<>(); + int count = randomIntBetween(1, this.indices.size()); + while (uniqueIndices.size() < count) { + uniqueIndices.add(randomFrom(this.indices)); + } + String[] indices = new String[count]; + int i = 0; + for (String index : uniqueIndices) { + indices[i++] = randomBoolean() ? index + "-alias" : index; + } + return indices; + } + + protected static void assertSameIndices(IndicesRequest originalRequest, String... actions) { + assertSameIndices(originalRequest, false, actions); + } + + protected static void assertSameIndicesOptionalRequests(IndicesRequest originalRequest, + String... actions) { + assertSameIndices(originalRequest, true, actions); + } + + protected static void assertSameIndices(IndicesRequest originalRequest, boolean optional, + String... actions) { + for (String action : actions) { + List requests = consumeTransportRequests(action); + if (!optional) { + assertThat("no internal requests intercepted for action [" + action + "]", + requests.size(), greaterThan(0)); + } + for (TransportRequest internalRequest : requests) { + IndicesRequest indicesRequest = convertRequest(internalRequest); + assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), + equalTo(originalRequest.indices())); + assertThat(indicesRequest.indicesOptions(), + equalTo(originalRequest.indicesOptions())); + } + } + } + protected static void assertIndicesSubset(List indices, String... actions) { + //indices returned by each bulk shard request need to be a subset of the original indices + for (String action : actions) { + List requests = consumeTransportRequests(action); + assertThat("no internal requests intercepted for action [" + action + "]", + requests.size(), greaterThan(0)); + for (TransportRequest internalRequest : requests) { + IndicesRequest indicesRequest = convertRequest(internalRequest); + for (String index : indicesRequest.indices()) { + assertThat(indices, hasItem(index)); + } + } + } + } + private static IndicesRequest convertRequest(TransportRequest request) { + final IndicesRequest indicesRequest; + if (request instanceof IndicesRequest) { + indicesRequest = (IndicesRequest) request; + } else { + indicesRequest = resolveRequest(request); + } + return indicesRequest; + } + private static > R resolveRequest( + TransportRequest requestOrWrappedRequest) { + if (requestOrWrappedRequest instanceof TransportReplicationAction.ConcreteShardRequest) { + requestOrWrappedRequest = + ((TransportReplicationAction.ConcreteShardRequest) requestOrWrappedRequest) + .getRequest(); + } + return (R) requestOrWrappedRequest; + } + + protected static void assertAllRequestsHaveBeenConsumed() { + withInterceptingTransportService(its -> + assertThat(its.requests.entrySet(), emptyIterable())); + } + + protected static void clearInterceptedActions() { + withInterceptingTransportService(InterceptingTransportService::clearInterceptedActions); + } + + protected static void interceptTransportActions(String... actions) { + withInterceptingTransportService(its -> its.interceptTransportActions(actions)); + } + + protected static List consumeTransportRequests(String action) { + List requests = new ArrayList<>(); + withInterceptingTransportService(its -> { + List transportRequests = its.consumeRequests(action); + if (transportRequests != null) { + requests.addAll(transportRequests); + } + }); + return requests; + } + + private static void withInterceptingTransportService( + Consumer callback) { + for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { + InterceptingTransportService its = pluginsService + .filterPlugins(InterceptingTransportService.TestPlugin.class) + .stream().findFirst().get().instance; + callback.accept(its); + } + + } + + private static class InterceptingTransportService implements TransportInterceptor { + public static class TestPlugin extends Plugin implements NetworkPlugin { + public final InterceptingTransportService instance = new InterceptingTransportService(); + @Override + public List getTransportInterceptors( + NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) { + return Collections.singletonList(instance); + } + } + + private final Set actions = new HashSet<>(); + + private final Map> requests = new HashMap<>(); + + @Override + public TransportRequestHandler interceptHandler( + String action, String executor, boolean forceExecution, + TransportRequestHandler actualHandler) { + return new InterceptingRequestHandler<>(action, actualHandler); + } + + synchronized List consumeRequests(String action) { + return requests.remove(action); + } + + synchronized void interceptTransportActions(String... actions) { + Collections.addAll(this.actions, actions); + } + + synchronized void clearInterceptedActions() { + actions.clear(); + } + + + private class InterceptingRequestHandler + implements TransportRequestHandler { + private final TransportRequestHandler requestHandler; + private final String action; + + InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { + this.requestHandler = requestHandler; + this.action = action; + } + + @Override + public void messageReceived(T request, TransportChannel channel, Task task) + throws Exception { + synchronized (InterceptingTransportService.this) { + if (actions.contains(action)) { + List requestList = requests.get(action); + if (requestList == null) { + requestList = new ArrayList<>(); + requestList.add(request); + requests.put(action, requestList); + } else { + requestList.add(request); + } + } + } + requestHandler.messageReceived(request, channel, task); + } + + @Override + public void messageReceived(T request, TransportChannel channel) throws Exception { + messageReceived(request, channel, null); + } + } + } +} From 047173bb7690e8737b64d66d19f7042a9c25e320 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 16:27:39 -0400 Subject: [PATCH 20/28] Hack transportCient so this works --- .../client/transport/TransportClient.java | 17 ++++++++++++++--- .../analysis/common/AnalyzeActionTests.java | 8 ++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java index 01fdf98ceb427..ce86c95a7f659 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -26,6 +26,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -45,8 +47,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.node.Node; import org.elasticsearch.node.InternalSettingsPreparer; +import org.elasticsearch.node.Node; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -191,8 +193,17 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings final TransportClientNodesService nodesService = new TransportClientNodesService(settings, transportService, threadPool, failureListner == null ? (t, e) -> {} : failureListner); - final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService, - actionModule.getActions().values().stream().map(x -> x.getAction()).collect(Collectors.toList())); + + @SuppressWarnings("rawtypes") + List actionsToProxy = new ArrayList<>(); + actionsToProxy.addAll(actionModule.getActions().values().stream() + .map(x -> x.getAction()) + .collect(toList())); + /* TransportAnalyzeAction lives in analysis-common but we still want it exposed over the + * core transport client. To do this we need sneak AnalyzeAction into the proxy list. */ + actionsToProxy.add(AnalyzeAction.INSTANCE); + final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, + nodesService, actionsToProxy); List pluginLifecycleComponents = new ArrayList<>(); pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream() diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java index 7626334680893..7fe1dae698c31 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java @@ -22,13 +22,16 @@ import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.core.IsNull; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; import java.util.Map; +import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -38,6 +41,11 @@ public class AnalyzeActionTests extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return singletonList(CommonAnalysisPlugin.class); + } + public void testSimpleAnalyzerTests() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); From 5fb14429f3b04743baa6969ab461797471e5b57f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 16:43:25 -0400 Subject: [PATCH 21/28] Let's not --- .../resources/checkstyle_suppressions.xml | 5 + .../elasticsearch/action/ActionModule.java | 5 + .../indices/analyze/AnalyzeResponse.java | 5 +- .../analyze}/TransportAnalyzeAction.java | 270 +++++--------- .../client/transport/TransportClient.java | 17 +- .../admin/indices}/RestAnalyzeAction.java | 45 +-- .../action/IndicesRequestIT.java | 251 ++++++++++++- .../indices/TransportAnalyzeActionTests.java | 280 ++++++++++++++ .../TransportReplicationActionTests.java | 14 + .../elasticsearch/cluster/NoMasterNodeIT.java | 9 + .../indices/analyze/AnalyzeActionIT.java | 344 ++++++------------ .../indices}/RestAnalyzeActionTests.java | 37 +- .../analysis/common/CommonAnalysisPlugin.java | 32 +- .../analysis/common/AnalyzeRequestTests.java | 55 --- .../common/TransportAnalyzeActionTests.java | 2 + .../action/IndicesRequestTestCase.java | 276 -------------- 16 files changed, 798 insertions(+), 849 deletions(-) rename {modules/analysis-common/src/main/java/org/elasticsearch/analysis/common => core/src/main/java/org/elasticsearch/action/admin/indices/analyze}/TransportAnalyzeAction.java (70%) rename {modules/analysis-common/src/main/java/org/elasticsearch/analysis/common => core/src/main/java/org/elasticsearch/rest/action/admin/indices}/RestAnalyzeAction.java (83%) create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java rename modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java => core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java (79%) rename {modules/analysis-common/src/test/java/org/elasticsearch/analysis/common => core/src/test/java/org/elasticsearch/rest/action/admin/indices}/RestAnalyzeActionTests.java (88%) delete mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 1d052b519e92a..f81d8d32946a3 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -245,6 +245,7 @@ + @@ -1630,6 +1631,7 @@ + @@ -2256,6 +2258,7 @@ + @@ -2928,6 +2931,7 @@ + @@ -3010,6 +3014,7 @@ + diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index 08a22ba2c44f9..c1d0541d4ce10 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -79,6 +79,8 @@ import org.elasticsearch.action.admin.indices.alias.exists.TransportAliasesExistAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.TransportGetAliasesAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.cache.clear.TransportClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.close.CloseIndexAction; @@ -236,6 +238,7 @@ import org.elasticsearch.rest.action.admin.cluster.RestRestoreSnapshotAction; import org.elasticsearch.rest.action.admin.cluster.RestSnapshotsStatusAction; import org.elasticsearch.rest.action.admin.cluster.RestVerifyRepositoryAction; +import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction; import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction; import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; @@ -439,6 +442,7 @@ public void reg actions.register(PutMappingAction.INSTANCE, TransportPutMappingAction.class); actions.register(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class); actions.register(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class); + actions.register(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class); actions.register(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class); actions.register(GetIndexTemplatesAction.INSTANCE, TransportGetIndexTemplatesAction.class); actions.register(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class); @@ -545,6 +549,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestUpdateSettingsAction(settings, restController)); registerHandler.accept(new RestGetSettingsAction(settings, restController, indexScopedSettings, settingsFilter)); + registerHandler.accept(new RestAnalyzeAction(settings, restController)); registerHandler.accept(new RestGetIndexTemplateAction(settings, restController)); registerHandler.accept(new RestPutIndexTemplateAction(settings, restController)); registerHandler.accept(new RestDeleteIndexTemplateAction(settings, restController)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index a38b1b4b43145..e4c6b34d9c7d7 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -151,10 +151,7 @@ public void writeTo(StreamOutput out) throws IOException { private List tokens; - /** - * For deserialization. - */ - public AnalyzeResponse() { + AnalyzeResponse() { } public AnalyzeResponse(List tokens, DetailAnalyzeResponse detail) { diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java similarity index 70% rename from modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 41b899981fb84..d7e299b1cf1b5 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/TransportAnalyzeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.analysis.common; +package org.elasticsearch.action.admin.indices.analyze; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -30,10 +30,6 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.analyze.DetailAnalyzeResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.single.shard.TransportSingleShardAction; import org.elasticsearch.cluster.ClusterState; @@ -75,28 +71,19 @@ import java.util.Set; import java.util.TreeMap; -import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER; -import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_FILTER; -import static org.elasticsearch.index.analysis.AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER; -import static org.elasticsearch.index.analysis.AnalysisRegistry.getSettingsFromIndexSettings; - /** - * Transport action used to execute analyze requests. + * Transport action used to execute analyze requests */ -public class TransportAnalyzeAction extends - TransportSingleShardAction { +public class TransportAnalyzeAction extends TransportSingleShardAction { private final IndicesService indicesService; private final Environment environment; @Inject - public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, - ClusterService clusterService, TransportService transportService, - IndicesService indicesService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) { - super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, - actionFilters, indexNameExpressionResolver, AnalyzeRequest::new, - ThreadPool.Names.INDEX); + public TransportAnalyzeAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService, + IndicesService indicesService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Environment environment) { + super(settings, AnalyzeAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, AnalyzeRequest::new, ThreadPool.Names.INDEX); this.indicesService = indicesService; this.environment = environment; } @@ -141,8 +128,7 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId Analyzer analyzer = null; if (request.field() != null) { if (indexService == null) { - throw new IllegalArgumentException("No index provided, and trying to analyzer " - + "based on a specific field which requires the index parameter"); + throw new IllegalArgumentException("No index provided, and trying to analyzer based on a specific field which requires the index parameter"); } MappedFieldType fieldType = indexService.mapperService().fullName(request.field()); if (fieldType != null) { @@ -155,8 +141,7 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId analyzer = fieldType.indexAnalyzer(); } } else { - throw new IllegalArgumentException("Can't process field [" + request.field() - + "], Analysis requests are only supported on tokenized fields"); + throw new IllegalArgumentException("Can't process field [" + request.field() + "], Analysis requests are only supported on tokenized fields"); } field = fieldType.name(); } @@ -169,48 +154,40 @@ protected AnalyzeResponse shardOperation(AnalyzeRequest request, ShardId shardId } } final AnalysisRegistry analysisRegistry = indicesService.getAnalysis(); - IndexAnalyzers analzyers = - indexService == null ? null : indexService.getIndexAnalyzers(); - return analyze(request, field, analyzer, analzyers, analysisRegistry, environment); + return analyze(request, field, analyzer, indexService != null ? indexService.getIndexAnalyzers() : null, analysisRegistry, environment); } catch (IOException e) { throw new ElasticsearchException("analysis failed", e); } + } - public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, - IndexAnalyzers indexAnalyzers, AnalysisRegistry analysisRegistry, - Environment environment) throws IOException { + public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Analyzer analyzer, IndexAnalyzers indexAnalyzers, AnalysisRegistry analysisRegistry, Environment environment) throws IOException { + boolean closeAnalyzer = false; if (analyzer == null && request.analyzer() != null) { if (indexAnalyzers == null) { analyzer = analysisRegistry.getAnalyzer(request.analyzer()); if (analyzer == null) { - throw new IllegalArgumentException("failed to find global analyzer [" - + request.analyzer() + "]"); + throw new IllegalArgumentException("failed to find global analyzer [" + request.analyzer() + "]"); } } else { analyzer = indexAnalyzers.get(request.analyzer()); if (analyzer == null) { - throw new IllegalArgumentException("failed to find analyzer [" - + request.analyzer() + "]"); + throw new IllegalArgumentException("failed to find analyzer [" + request.analyzer() + "]"); } } + } else if (request.tokenizer() != null) { - final IndexSettings indexSettings = - indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings(); - TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers, - analysisRegistry, environment); + final IndexSettings indexSettings = indexAnalyzers == null ? null : indexAnalyzers.getIndexSettings(); + TokenizerFactory tokenizerFactory = parseTokenizerFactory(request, indexAnalyzers, analysisRegistry, environment); TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0]; - tokenFilterFactories = getTokenFilterFactories(request, indexSettings, analysisRegistry, - environment, tokenFilterFactories); + tokenFilterFactories = getTokenFilterFactories(request, indexSettings, analysisRegistry, environment, tokenFilterFactories); CharFilterFactory[] charFilterFactories = new CharFilterFactory[0]; - charFilterFactories = getCharFilterFactories(request, indexSettings, analysisRegistry, - environment, charFilterFactories); + charFilterFactories = getCharFilterFactories(request, indexSettings, analysisRegistry, environment, charFilterFactories); - analyzer = new CustomAnalyzer( - tokenizerFactory, charFilterFactories, tokenFilterFactories); + analyzer = new CustomAnalyzer(tokenizerFactory, charFilterFactories, tokenFilterFactories); closeAnalyzer = true; } else if (analyzer == null) { if (indexAnalyzers == null) { @@ -239,8 +216,7 @@ public static AnalyzeResponse analyze(AnalyzeRequest request, String field, Anal return new AnalyzeResponse(tokens, detail); } - private static List simpleAnalyze(AnalyzeRequest request, - Analyzer analyzer, String field) { + private static List simpleAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) { List tokens = new ArrayList<>(); int lastPosition = -1; int lastOffset = 0; @@ -248,8 +224,7 @@ private static List simpleAnalyze(AnalyzeRequest r try (TokenStream stream = analyzer.tokenStream(field, text)) { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); - PositionIncrementAttribute posIncr = stream.addAttribute( - PositionIncrementAttribute.class); + PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); @@ -259,9 +234,9 @@ private static List simpleAnalyze(AnalyzeRequest r if (increment > 0) { lastPosition = lastPosition + increment; } - tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, - lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), - posLen.getPositionLength(), type.type(), null)); + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), + lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null)); + } stream.end(); lastOffset += offset.endOffset(); @@ -276,8 +251,7 @@ private static List simpleAnalyze(AnalyzeRequest r return tokens; } - private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, - String field) { + private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analyzer analyzer, String field) { DetailAnalyzeResponse detailResponse; final Set includeAttributes = new HashSet<>(); if (request.attributes() != null) { @@ -289,8 +263,7 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy CustomAnalyzer customAnalyzer = null; if (analyzer instanceof CustomAnalyzer) { customAnalyzer = (CustomAnalyzer) analyzer; - } else if (analyzer instanceof NamedAnalyzer && - ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { + } else if (analyzer instanceof NamedAnalyzer && ((NamedAnalyzer) analyzer).analyzer() instanceof CustomAnalyzer) { customAnalyzer = (CustomAnalyzer) ((NamedAnalyzer) analyzer).analyzer(); } @@ -300,11 +273,8 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy TokenizerFactory tokenizerFactory = customAnalyzer.tokenizerFactory(); TokenFilterFactory[] tokenFilterFactories = customAnalyzer.tokenFilters(); - int charFilterLength = charFilterFactories == null ? 0 : charFilterFactories.length; - String[][] charFiltersTexts = new String[charFilterLength][request.text().length]; - int tokenFilterLength = tokenFilterFactories == null ? 0 : tokenFilterFactories.length; - TokenListCreator[] tokenFiltersTokenListCreator = - new TokenListCreator[tokenFilterLength]; + String[][] charFiltersTexts = new String[charFilterFactories != null ? charFilterFactories.length : 0][request.text().length]; + TokenListCreator[] tokenFiltersTokenListCreator = new TokenListCreator[tokenFilterFactories != null ? tokenFilterFactories.length : 0]; TokenListCreator tokenizerTokenListCreator = new TokenListCreator(); @@ -314,12 +284,10 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy Reader reader = new FastStringReader(charFilteredSource); if (charFilterFactories != null) { - for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; - charFilterIndex++) { + for (int charFilterIndex = 0; charFilterIndex < charFilterFactories.length; charFilterIndex++) { reader = charFilterFactories[charFilterIndex].create(reader); Reader readerForWriteOut = new FastStringReader(charFilteredSource); - readerForWriteOut = charFilterFactories[charFilterIndex] - .create(readerForWriteOut); + readerForWriteOut = charFilterFactories[charFilterIndex].create(readerForWriteOut); charFilteredSource = writeCharStream(readerForWriteOut); charFiltersTexts[charFilterIndex][textIndex] = charFilteredSource; } @@ -328,49 +296,36 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy // analyzing only tokenizer Tokenizer tokenizer = tokenizerFactory.create(); tokenizer.setReader(reader); - tokenizerTokenListCreator.analyze( - tokenizer, customAnalyzer, field, includeAttributes); + tokenizerTokenListCreator.analyze(tokenizer, customAnalyzer, field, includeAttributes); // analyzing each tokenfilter if (tokenFilterFactories != null) { - for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; - tokenFilterIndex++) { + for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFilterFactories.length; tokenFilterIndex++) { if (tokenFiltersTokenListCreator[tokenFilterIndex] == null) { tokenFiltersTokenListCreator[tokenFilterIndex] = new TokenListCreator(); } TokenStream stream = createStackedTokenStream(request.text()[textIndex], - charFilterFactories, tokenizerFactory, - tokenFilterFactories, tokenFilterIndex + 1); - tokenFiltersTokenListCreator[tokenFilterIndex].analyze( - stream, customAnalyzer, field, includeAttributes); + charFilterFactories, tokenizerFactory, tokenFilterFactories, tokenFilterIndex + 1); + tokenFiltersTokenListCreator[tokenFilterIndex].analyze(stream, customAnalyzer, field, includeAttributes); } } } - DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = - new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length]; + DetailAnalyzeResponse.CharFilteredText[] charFilteredLists = new DetailAnalyzeResponse.CharFilteredText[charFiltersTexts.length]; if (charFilterFactories != null) { - for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; - charFilterIndex++) { + for (int charFilterIndex = 0; charFilterIndex < charFiltersTexts.length; charFilterIndex++) { charFilteredLists[charFilterIndex] = new DetailAnalyzeResponse.CharFilteredText( - charFilterFactories[charFilterIndex].name(), - charFiltersTexts[charFilterIndex]); + charFilterFactories[charFilterIndex].name(), charFiltersTexts[charFilterIndex]); } } - DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = - new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length]; + DetailAnalyzeResponse.AnalyzeTokenList[] tokenFilterLists = new DetailAnalyzeResponse.AnalyzeTokenList[tokenFiltersTokenListCreator.length]; if (tokenFilterFactories != null) { - for (int tokenFilterIndex = 0; - tokenFilterIndex < tokenFiltersTokenListCreator.length; - tokenFilterIndex++) { + for (int tokenFilterIndex = 0; tokenFilterIndex < tokenFiltersTokenListCreator.length; tokenFilterIndex++) { tokenFilterLists[tokenFilterIndex] = new DetailAnalyzeResponse.AnalyzeTokenList( - tokenFilterFactories[tokenFilterIndex].name(), - tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens()); + tokenFilterFactories[tokenFilterIndex].name(), tokenFiltersTokenListCreator[tokenFilterIndex].getArrayTokens()); } } - detailResponse = new DetailAnalyzeResponse(charFilteredLists, - new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), - tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); + detailResponse = new DetailAnalyzeResponse(charFilteredLists, new DetailAnalyzeResponse.AnalyzeTokenList(tokenizerFactory.name(), tokenizerTokenListCreator.getArrayTokens()), tokenFilterLists); } else { String name; if (analyzer instanceof NamedAnalyzer) { @@ -384,15 +339,12 @@ private static DetailAnalyzeResponse detailAnalyze(AnalyzeRequest request, Analy tokenListCreator.analyze(analyzer.tokenStream(field, text), analyzer, field, includeAttributes); } - detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList( - name, tokenListCreator.getArrayTokens())); + detailResponse = new DetailAnalyzeResponse(new DetailAnalyzeResponse.AnalyzeTokenList(name, tokenListCreator.getArrayTokens())); } return detailResponse; } - private static TokenStream createStackedTokenStream(String source, - CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, - TokenFilterFactory[] tokenFilterFactories, int current) { + private static TokenStream createStackedTokenStream(String source, CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, TokenFilterFactory[] tokenFilterFactories, int current) { Reader reader = new FastStringReader(source); for (CharFilterFactory charFilterFactory : charFilterFactories) { reader = charFilterFactory.create(reader); @@ -433,13 +385,11 @@ private static class TokenListCreator { tokens = new ArrayList<>(); } - private void analyze(TokenStream stream, Analyzer analyzer, - String field, Set includeAttributes) { + private void analyze(TokenStream stream, Analyzer analyzer, String field, Set includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); - PositionIncrementAttribute posIncr = - stream.addAttribute(PositionIncrementAttribute.class); + PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); @@ -449,10 +399,8 @@ private void analyze(TokenStream stream, Analyzer analyzer, if (increment > 0) { lastPosition = lastPosition + increment; } - tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, - lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), - posLen.getPositionLength(), type.type(), - extractExtendedAttributes(stream, includeAttributes))); + tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), + lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); @@ -483,8 +431,7 @@ private AnalyzeResponse.AnalyzeToken[] getArrayTokens() { * @param includeAttributes filtering attributes * @return Map<key value> */ - private static Map extractExtendedAttributes(TokenStream stream, - final Set includeAttributes) { + private static Map extractExtendedAttributes(TokenStream stream, final Set includeAttributes) { final Map extendedAttributes = new TreeMap<>(); stream.reflectWith((attClass, key, value) -> { @@ -500,9 +447,7 @@ private static Map extractExtendedAttributes(TokenStream stream, if (TypeAttribute.class.isAssignableFrom(attClass)) { return; } - if (includeAttributes == null - || includeAttributes.isEmpty() - || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { + if (includeAttributes == null || includeAttributes.isEmpty() || includeAttributes.contains(key.toLowerCase(Locale.ROOT))) { if (value instanceof BytesRef) { final BytesRef p = (BytesRef) value; value = p.toString(); @@ -514,9 +459,8 @@ private static Map extractExtendedAttributes(TokenStream stream, return extendedAttributes; } - private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, - IndexSettings indexSettings, AnalysisRegistry analysisRegistry, Environment environment, - CharFilterFactory[] charFilterFactories) throws IOException { + private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry, + Environment environment, CharFilterFactory[] charFilterFactories) throws IOException { if (request.charFilters() != null && request.charFilters().size() > 0) { charFilterFactories = new CharFilterFactory[request.charFilters().size()]; for (int i = 0; i < request.charFilters().size(); i++) { @@ -526,55 +470,43 @@ private static CharFilterFactory[] getCharFilterFactories(AnalyzeRequest request Settings settings = getAnonymousSettings(charFilter.definition); String charFilterTypeName = settings.get("type"); if (charFilterTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous " - + "char filter: " + charFilter.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous char filter: " + charFilter.definition); } AnalysisModule.AnalysisProvider charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilterTypeName); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global char filter " - + "under [" + charFilterTypeName + "]"); + throw new IllegalArgumentException("failed to find global char filter under [" + charFilterTypeName + "]"); } // Need to set anonymous "name" of char_filter - charFilterFactories[i] = charFilterFactoryFactory.get( - getNaIndexSettings(settings), environment, - "_anonymous_charfilter_[" + i + "]", settings); + charFilterFactories[i] = charFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_charfilter_[" + i + "]", settings); } else { AnalysisModule.AnalysisProvider charFilterFactoryFactory; if (indexSettings == null) { - charFilterFactoryFactory = - analysisRegistry.getCharFilterProvider(charFilter.name); + charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global char filter" - + "under [" + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find global char filter under [" + charFilter.name + "]"); } - charFilterFactories[i] = charFilterFactoryFactory.get( - environment, charFilter.name); + charFilterFactories[i] = charFilterFactoryFactory.get(environment, charFilter.name); } else { - charFilterFactoryFactory = analysisRegistry.getCharFilterProvider( - charFilter.name, indexSettings); + charFilterFactoryFactory = analysisRegistry.getCharFilterProvider(charFilter.name, indexSettings); if (charFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find char filter under [" - + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]"); } - Settings analysisSettings = getSettingsFromIndexSettings( - indexSettings, INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name); - charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, - environment, charFilter.name, analysisSettings); + charFilterFactories[i] = charFilterFactoryFactory.get(indexSettings, environment, charFilter.name, + AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, + AnalysisRegistry.INDEX_ANALYSIS_CHAR_FILTER + "." + charFilter.name)); } } if (charFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find char filter under [" - + charFilter.name + "]"); + throw new IllegalArgumentException("failed to find char filter under [" + charFilter.name + "]"); } } } return charFilterFactories; } - private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, - IndexSettings indexSettings, AnalysisRegistry analysisRegistry, Environment environment, - TokenFilterFactory[] tokenFilterFactories) throws IOException { + private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest request, IndexSettings indexSettings, AnalysisRegistry analysisRegistry, + Environment environment, TokenFilterFactory[] tokenFilterFactories) throws IOException { if (request.tokenFilters() != null && request.tokenFilters().size() > 0) { tokenFilterFactories = new TokenFilterFactory[request.tokenFilters().size()]; for (int i = 0; i < request.tokenFilters().size(); i++) { @@ -584,55 +516,43 @@ private static TokenFilterFactory[] getTokenFilterFactories(AnalyzeRequest reque Settings settings = getAnonymousSettings(tokenFilter.definition); String filterTypeName = settings.get("type"); if (filterTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous " - + "token filter: " + tokenFilter.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous token filter: " + tokenFilter.definition); } AnalysisModule.AnalysisProvider tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(filterTypeName); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global token filter " - + "under [" + filterTypeName + "]"); + throw new IllegalArgumentException("failed to find global token filter under [" + filterTypeName + "]"); } // Need to set anonymous "name" of tokenfilter - tokenFilterFactories[i] = tokenFilterFactoryFactory.get( - getNaIndexSettings(settings), environment, - "_anonymous_tokenfilter_[" + i + "]", settings); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenfilter_[" + i + "]", settings); } else { AnalysisModule.AnalysisProvider tokenFilterFactoryFactory; if (indexSettings == null) { - tokenFilterFactoryFactory = - analysisRegistry.getTokenFilterProvider(tokenFilter.name); + tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global token filter " - + "under [" + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find global token filter under [" + tokenFilter.name + "]"); } - tokenFilterFactories[i] = tokenFilterFactoryFactory.get( - environment, tokenFilter.name); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get(environment, tokenFilter.name); } else { - tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider( - tokenFilter.name, indexSettings); + tokenFilterFactoryFactory = analysisRegistry.getTokenFilterProvider(tokenFilter.name, indexSettings); if (tokenFilterFactoryFactory == null) { - throw new IllegalArgumentException("failed to find token filter under [" - + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find token filter under [" + tokenFilter.name + "]"); } - tokenFilterFactories[i] = tokenFilterFactoryFactory.get( - indexSettings, environment, tokenFilter.name, - getSettingsFromIndexSettings(indexSettings, - INDEX_ANALYSIS_FILTER + "." + tokenFilter.name)); + tokenFilterFactories[i] = tokenFilterFactoryFactory.get(indexSettings, environment, tokenFilter.name, + AnalysisRegistry.getSettingsFromIndexSettings(indexSettings, + AnalysisRegistry.INDEX_ANALYSIS_FILTER + "." + tokenFilter.name)); } } if (tokenFilterFactories[i] == null) { - throw new IllegalArgumentException("failed to find or create token filter " - + "under [" + tokenFilter.name + "]"); + throw new IllegalArgumentException("failed to find or create token filter under [" + tokenFilter.name + "]"); } } } return tokenFilterFactories; } - private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, - IndexAnalyzers indexAnalzyers, AnalysisRegistry analysisRegistry, - Environment environment) throws IOException { + private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, IndexAnalyzers indexAnalzyers, + AnalysisRegistry analysisRegistry, Environment environment) throws IOException { TokenizerFactory tokenizerFactory; final AnalyzeRequest.NameOrDefinition tokenizer = request.tokenizer(); // parse anonymous settings @@ -640,46 +560,38 @@ private static TokenizerFactory parseTokenizerFactory(AnalyzeRequest request, Settings settings = getAnonymousSettings(tokenizer.definition); String tokenizerTypeName = settings.get("type"); if (tokenizerTypeName == null) { - throw new IllegalArgumentException("Missing [type] setting for anonymous " - + "tokenizer: " + tokenizer.definition); + throw new IllegalArgumentException("Missing [type] setting for anonymous tokenizer: " + tokenizer.definition); } AnalysisModule.AnalysisProvider tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizerTypeName); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global tokenizer under [" - + tokenizerTypeName + "]"); + throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizerTypeName + "]"); } // Need to set anonymous "name" of tokenizer - tokenizerFactory = tokenizerFactoryFactory.get( - getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings); + tokenizerFactory = tokenizerFactoryFactory.get(getNaIndexSettings(settings), environment, "_anonymous_tokenizer", settings); } else { AnalysisModule.AnalysisProvider tokenizerFactoryFactory; if (indexAnalzyers == null) { tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find global tokenizer under [" - + tokenizer.name + "]"); + throw new IllegalArgumentException("failed to find global tokenizer under [" + tokenizer.name + "]"); } tokenizerFactory = tokenizerFactoryFactory.get(environment, tokenizer.name); } else { - tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider( - tokenizer.name, indexAnalzyers.getIndexSettings()); + tokenizerFactoryFactory = analysisRegistry.getTokenizerProvider(tokenizer.name, indexAnalzyers.getIndexSettings()); if (tokenizerFactoryFactory == null) { - throw new IllegalArgumentException("failed to find tokenizer under [" - + tokenizer.name + "]"); + throw new IllegalArgumentException("failed to find tokenizer under [" + tokenizer.name + "]"); } - tokenizerFactory = tokenizerFactoryFactory.get( - indexAnalzyers.getIndexSettings(), environment, tokenizer.name, - getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(), - INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name)); + tokenizerFactory = tokenizerFactoryFactory.get(indexAnalzyers.getIndexSettings(), environment, tokenizer.name, + AnalysisRegistry.getSettingsFromIndexSettings(indexAnalzyers.getIndexSettings(), + AnalysisRegistry.INDEX_ANALYSIS_TOKENIZER + "." + tokenizer.name)); } } return tokenizerFactory; } private static IndexSettings getNaIndexSettings(Settings settings) { - IndexMetaData metaData = IndexMetaData - .builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build(); + IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build(); return new IndexSettings(metaData, Settings.EMPTY); } diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java index ce86c95a7f659..01fdf98ceb427 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java @@ -26,8 +26,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.GenericAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -47,8 +45,8 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; +import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -193,17 +191,8 @@ private static ClientTemplate buildTemplate(Settings providedSettings, Settings final TransportClientNodesService nodesService = new TransportClientNodesService(settings, transportService, threadPool, failureListner == null ? (t, e) -> {} : failureListner); - - @SuppressWarnings("rawtypes") - List actionsToProxy = new ArrayList<>(); - actionsToProxy.addAll(actionModule.getActions().values().stream() - .map(x -> x.getAction()) - .collect(toList())); - /* TransportAnalyzeAction lives in analysis-common but we still want it exposed over the - * core transport client. To do this we need sneak AnalyzeAction into the proxy list. */ - actionsToProxy.add(AnalyzeAction.INSTANCE); - final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, - nodesService, actionsToProxy); + final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService, + actionModule.getActions().values().stream().map(x -> x.getAction()).collect(Collectors.toList())); List pluginLifecycleComponents = new ArrayList<>(); pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream() diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java similarity index 83% rename from modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java rename to core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java index 9c4bfe859fd82..52ba58e4c38c2 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/RestAnalyzeAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.analysis.common; +package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.client.node.NodeClient; @@ -57,8 +57,8 @@ public RestAnalyzeAction(Settings settings, RestController controller) { } @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) - throws IOException { + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index")); try (XContentParser parser = request.contentOrSourceParamParser()) { @@ -67,8 +67,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC throw new IllegalArgumentException("Failed to parse request body", e); } - return channel -> client.admin().indices().analyze( - analyzeRequest, new RestToXContentListener<>(channel)); + return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel)); } static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest) @@ -81,25 +80,20 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); - } else if (Fields.TEXT.match(currentFieldName) - && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.text(parser.text()); - } else if (Fields.TEXT.match(currentFieldName) - && token == XContentParser.Token.START_ARRAY) { + } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { List texts = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { - throw new IllegalArgumentException(currentFieldName - + " array element should only contain text"); + throw new IllegalArgumentException(currentFieldName + " array element should only contain text"); } texts.add(parser.text()); } analyzeRequest.text(texts.toArray(new String[texts.size()])); - } else if (Fields.ANALYZER.match(currentFieldName) - && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.ANALYZER.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.analyzer(parser.text()); - } else if (Fields.FIELD.match(currentFieldName) - && token == XContentParser.Token.VALUE_STRING) { + } else if (Fields.FIELD.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) { analyzeRequest.field(parser.text()); } else if (Fields.TOKENIZER.match(currentFieldName)) { if (token == XContentParser.Token.VALUE_STRING) { @@ -107,8 +101,7 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques } else if (token == XContentParser.Token.START_OBJECT) { analyzeRequest.tokenizer(parser.map()); } else { - throw new IllegalArgumentException(currentFieldName - + " should be tokenizer's name or setting"); + throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting"); } } else if (Fields.TOKEN_FILTERS.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { @@ -130,32 +123,28 @@ static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeReques } else if (token == XContentParser.Token.START_OBJECT) { analyzeRequest.addCharFilter(parser.map()); } else { - throw new IllegalArgumentException(currentFieldName + " array element " - + "should contain char filter's name or setting"); + throw new IllegalArgumentException(currentFieldName + + " array element should contain char filter's name or setting"); } } } else if (Fields.EXPLAIN.match(currentFieldName)) { if (parser.isBooleanValue()) { analyzeRequest.explain(parser.booleanValue()); } else { - throw new IllegalArgumentException(currentFieldName - + " must be either 'true' or 'false'"); + throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'"); } - } else if (Fields.ATTRIBUTES.match(currentFieldName) - && token == XContentParser.Token.START_ARRAY) { + } else if (Fields.ATTRIBUTES.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) { List attributes = new ArrayList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token.isValue() == false) { - throw new IllegalArgumentException(currentFieldName - + " array element should only contain attribute name"); + throw new IllegalArgumentException(currentFieldName + " array element should only contain attribute name"); } attributes.add(parser.text()); } analyzeRequest.attributes(attributes.toArray(new String[attributes.size()])); } else { - throw new IllegalArgumentException("Unknown parameter [" + currentFieldName - + "] in request body or parameter is of the wrong type[" - + token + "] "); + throw new IllegalArgumentException("Unknown parameter [" + + currentFieldName + "] in request body or parameter is of the wrong type[" + token + "] "); } } } diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java index 51d97bc01cb30..0f3812c0cd6ba 100644 --- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java @@ -19,6 +19,9 @@ package org.elasticsearch.action; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheAction; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexAction; @@ -63,8 +66,8 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.replication.TransportReplicationActionTests; import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.action.termvectors.MultiTermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsAction; @@ -74,29 +77,51 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; +import org.elasticsearch.action.search.SearchTransportService; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportInterceptor; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.TransportRequestHandler; +import org.junit.After; +import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasItem; @ClusterScope(scope = Scope.SUITE, numClientNodes = 1, minNumDataNodes = 2) -public class IndicesRequestIT extends IndicesRequestTestCase { +public class IndicesRequestIT extends ESIntegTestCase { + + private final List indices = new ArrayList<>(); + @Override protected int minimumNumberOfShards() { //makes sure that a reduce is always needed when searching @@ -121,10 +146,7 @@ protected Settings nodeSettings(int ordinal) { @Override protected Collection> nodePlugins() { - List> plugins = new ArrayList<>(); - plugins.addAll(super.nodePlugins()); - plugins.add(CustomScriptPlugin.class); - return plugins; + return Arrays.asList(InterceptingTransportService.TestPlugin.class, CustomScriptPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -136,6 +158,24 @@ protected Map, Object>> pluginScripts() { } } + @Before + public void setup() { + int numIndices = iterations(1, 5); + for (int i = 0; i < numIndices; i++) { + indices.add("test" + i); + } + for (String index : indices) { + assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias"))); + } + ensureGreen(); + } + + @After + public void cleanUp() { + assertAllRequestsHaveBeenConsumed(); + indices.clear(); + } + public void testGetFieldMappings() { String getFieldMappingsShardAction = GetFieldMappingsAction.NAME + "[index][s]"; interceptTransportActions(getFieldMappingsShardAction); @@ -148,6 +188,18 @@ public void testGetFieldMappings() { assertSameIndices(getFieldMappingsRequest, getFieldMappingsShardAction); } + public void testAnalyze() { + String analyzeShardAction = AnalyzeAction.NAME + "[s]"; + interceptTransportActions(analyzeShardAction); + + AnalyzeRequest analyzeRequest = new AnalyzeRequest(randomIndexOrAlias()); + analyzeRequest.text("text"); + internalCluster().coordOnlyNodeClient().admin().indices().analyze(analyzeRequest).actionGet(); + + clearInterceptedActions(); + assertSameIndices(analyzeRequest, analyzeShardAction); + } + public void testIndex() { String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"}; interceptTransportActions(indexShardActions); @@ -527,4 +579,191 @@ public void testSearchDfsQueryThenFetch() throws Exception { //free context messages are not necessarily sent, but if they are, check their indices assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME); } + + private static void assertSameIndices(IndicesRequest originalRequest, String... actions) { + assertSameIndices(originalRequest, false, actions); + } + + private static void assertSameIndicesOptionalRequests(IndicesRequest originalRequest, String... actions) { + assertSameIndices(originalRequest, true, actions); + } + + private static void assertSameIndices(IndicesRequest originalRequest, boolean optional, String... actions) { + for (String action : actions) { + List requests = consumeTransportRequests(action); + if (!optional) { + assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); + } + for (TransportRequest internalRequest : requests) { + IndicesRequest indicesRequest = convertRequest(internalRequest); + assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), equalTo(originalRequest.indices())); + assertThat(indicesRequest.indicesOptions(), equalTo(originalRequest.indicesOptions())); + } + } + } + private static void assertIndicesSubset(List indices, String... actions) { + //indices returned by each bulk shard request need to be a subset of the original indices + for (String action : actions) { + List requests = consumeTransportRequests(action); + assertThat("no internal requests intercepted for action [" + action + "]", requests.size(), greaterThan(0)); + for (TransportRequest internalRequest : requests) { + IndicesRequest indicesRequest = convertRequest(internalRequest); + for (String index : indicesRequest.indices()) { + assertThat(indices, hasItem(index)); + } + } + } + } + + static IndicesRequest convertRequest(TransportRequest request) { + final IndicesRequest indicesRequest; + if (request instanceof IndicesRequest) { + indicesRequest = (IndicesRequest) request; + } else { + indicesRequest = TransportReplicationActionTests.resolveRequest(request); + } + return indicesRequest; + } + + private String randomIndexOrAlias() { + String index = randomFrom(indices); + if (randomBoolean()) { + return index + "-alias"; + } else { + return index; + } + } + + private String[] randomIndicesOrAliases() { + int count = randomIntBetween(1, indices.size() * 2); //every index has an alias + String[] indices = new String[count]; + for (int i = 0; i < count; i++) { + indices[i] = randomIndexOrAlias(); + } + return indices; + } + + private String[] randomUniqueIndicesOrAliases() { + Set uniqueIndices = new HashSet<>(); + int count = randomIntBetween(1, this.indices.size()); + while (uniqueIndices.size() < count) { + uniqueIndices.add(randomFrom(this.indices)); + } + String[] indices = new String[count]; + int i = 0; + for (String index : uniqueIndices) { + indices[i++] = randomBoolean() ? index + "-alias" : index; + } + return indices; + } + + private static void assertAllRequestsHaveBeenConsumed() { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + Set>> entries = + pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() + .instance.requests.entrySet(); + assertThat(entries, emptyIterable()); + + } + } + + private static void clearInterceptedActions() { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() + .instance.clearInterceptedActions(); + } + } + + private static void interceptTransportActions(String... actions) { + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class).stream().findFirst().get() + .instance.interceptTransportActions(actions); + } + } + + private static List consumeTransportRequests(String action) { + List requests = new ArrayList<>(); + + Iterable pluginsServices = internalCluster().getInstances(PluginsService.class); + for (PluginsService pluginsService : pluginsServices) { + List transportRequests = pluginsService.filterPlugins(InterceptingTransportService.TestPlugin.class) + .stream().findFirst().get().instance.consumeRequests(action); + if (transportRequests != null) { + requests.addAll(transportRequests); + } + } + return requests; + } + + public static class InterceptingTransportService implements TransportInterceptor { + + public static class TestPlugin extends Plugin implements NetworkPlugin { + public final InterceptingTransportService instance = new InterceptingTransportService(); + @Override + public List getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry, + ThreadContext threadContext) { + return Collections.singletonList(instance); + } + } + + private final Set actions = new HashSet<>(); + + private final Map> requests = new HashMap<>(); + + @Override + public TransportRequestHandler interceptHandler(String action, String executor, + boolean forceExecution, + TransportRequestHandler actualHandler) { + return new InterceptingRequestHandler<>(action, actualHandler); + } + + synchronized List consumeRequests(String action) { + return requests.remove(action); + } + + synchronized void interceptTransportActions(String... actions) { + Collections.addAll(this.actions, actions); + } + + synchronized void clearInterceptedActions() { + actions.clear(); + } + + + private class InterceptingRequestHandler implements TransportRequestHandler { + + private final TransportRequestHandler requestHandler; + private final String action; + + InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { + this.requestHandler = requestHandler; + this.action = action; + } + + @Override + public void messageReceived(T request, TransportChannel channel, Task task) throws Exception { + synchronized (InterceptingTransportService.this) { + if (actions.contains(action)) { + List requestList = requests.get(action); + if (requestList == null) { + requestList = new ArrayList<>(); + requestList.add(request); + requests.put(action, requestList); + } else { + requestList.add(request); + } + } + } + requestHandler.messageReceived(request, channel, task); + } + + @Override + public void messageReceived(T request, TransportChannel channel) throws Exception { + messageReceived(request, channel, null); + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java new file mode 100644 index 0000000000000..57a83b2c68081 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java @@ -0,0 +1,280 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.admin.indices; + +import org.apache.lucene.analysis.MockTokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.index.mapper.AllFieldMapper; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; + +/** + * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the + * {@code common-analysis} module. + */ +public class TransportAnalyzeActionTests extends ESTestCase { + + private IndexAnalyzers indexAnalyzers; + private AnalysisRegistry registry; + private Environment environment; + + @Override + public void setUp() throws Exception { + super.setUp(); + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + + Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard") + .put("index.analysis.analyzer.custom_analyzer.filter", "mock").build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); + environment = new Environment(settings); + AnalysisPlugin plugin = new AnalysisPlugin() { + class MockFactory extends AbstractTokenFilterFactory { + MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); + } + + @Override + public TokenStream create(TokenStream tokenStream) { + return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET); + } + } + + @Override + public Map> getTokenFilters() { + return singletonMap("mock", MockFactory::new); + } + }; + registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry(); + indexAnalyzers = registry.build(idxSettings); + } + + public void testNoIndexAnalyzers() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the quick brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.addTokenFilter("lowercase"); + request.addTokenFilter("word_delimiter"); + request.text("the qu1ck brown fox"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + + request.analyzer(null); + request.tokenizer("whitespace"); + request.addCharFilter("html_strip"); + request.addTokenFilter("lowercase"); + request.addTokenFilter("word_delimiter"); + request.text("

the qu1ck brown fox

"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment); + tokens = analyze.getTokens(); + assertEquals(6, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("qu", tokens.get(1).getTerm()); + assertEquals("1", tokens.get(2).getTerm()); + assertEquals("ck", tokens.get(3).getTerm()); + assertEquals("brown", tokens.get(4).getTerm()); + assertEquals("fox", tokens.get(5).getTerm()); + } + + public void testFillsAttributes() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.analyzer("standard"); + request.text("the 1 brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals(0, tokens.get(0).getStartOffset()); + assertEquals(3, tokens.get(0).getEndOffset()); + assertEquals(0, tokens.get(0).getPosition()); + assertEquals("", tokens.get(0).getType()); + + assertEquals("1", tokens.get(1).getTerm()); + assertEquals(4, tokens.get(1).getStartOffset()); + assertEquals(5, tokens.get(1).getEndOffset()); + assertEquals(1, tokens.get(1).getPosition()); + assertEquals("", tokens.get(1).getType()); + + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals(6, tokens.get(2).getStartOffset()); + assertEquals(11, tokens.get(2).getEndOffset()); + assertEquals(2, tokens.get(2).getPosition()); + assertEquals("", tokens.get(2).getType()); + + assertEquals("fox", tokens.get(3).getTerm()); + assertEquals(12, tokens.get(3).getStartOffset()); + assertEquals(15, tokens.get(3).getEndOffset()); + assertEquals(3, tokens.get(3).getPosition()); + assertEquals("", tokens.get(3).getType()); + } + + public void testWithIndexAnalyzers() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.text("the quick brown fox"); + request.analyzer("custom_analyzer"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + List tokens = analyze.getTokens(); + assertEquals(3, tokens.size()); + assertEquals("quick", tokens.get(0).getTerm()); + assertEquals("brown", tokens.get(1).getTerm()); + assertEquals("fox", tokens.get(2).getTerm()); + + request.analyzer("standard"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("quick", tokens.get(1).getTerm()); + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals("fox", tokens.get(3).getTerm()); + + // Switch the analyzer out for just a tokenizer + request.analyzer(null); + request.tokenizer("standard"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(4, tokens.size()); + assertEquals("the", tokens.get(0).getTerm()); + assertEquals("quick", tokens.get(1).getTerm()); + assertEquals("brown", tokens.get(2).getTerm()); + assertEquals("fox", tokens.get(3).getTerm()); + + // Now try applying our token filter + request.addTokenFilter("mock"); + analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + tokens = analyze.getTokens(); + assertEquals(3, tokens.size()); + assertEquals("quick", tokens.get(0).getTerm()); + assertEquals("brown", tokens.get(1).getTerm()); + assertEquals("fox", tokens.get(2).getTerm()); + } + + public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .analyzer("custom_analyzer") + .text("the qu1ck brown fox-dog"), + AllFieldMapper.NAME, null, null, registry, environment)); + assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]"); + } + + public void testUnknown() throws IOException { + boolean notGlobal = randomBoolean(); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .analyzer("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global analyzer [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global tokenizer under [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("whitespace") + .addTokenFilter("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global token filter under [foobar]"); + } + + e = expectThrows(IllegalArgumentException.class, + () -> TransportAnalyzeAction.analyze( + new AnalyzeRequest() + .tokenizer("whitespace") + .addTokenFilter("lowercase") + .addCharFilter("foobar") + .text("the qu1ck brown fox"), + AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, registry, environment)); + if (notGlobal) { + assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); + } else { + assertEquals(e.getMessage(), "failed to find global char filter under [foobar]"); + } + } + + public void testNonPreBuildTokenFilter() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); + request.tokenizer("whitespace"); + request.addTokenFilter("min_hash"); + request.text("the quick brown fox"); + AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment); + List tokens = analyze.getTokens(); + int default_hash_count = 1; + int default_bucket_size = 512; + int default_hash_set_size = 1; + assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size()); + + } +} diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 3834d44f3ec5c..1bd943704707d 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -116,6 +116,20 @@ import static org.mockito.Mockito.when; public class TransportReplicationActionTests extends ESTestCase { + + /** + * takes a request that was sent by a {@link TransportReplicationAction} and captured + * and returns the underlying request if it's wrapped or the original (cast to the expected type). + * + * This will throw a {@link ClassCastException} if the request is of the wrong type. + */ + public static R resolveRequest(TransportRequest requestOrWrappedRequest) { + if (requestOrWrappedRequest instanceof TransportReplicationAction.ConcreteShardRequest) { + requestOrWrappedRequest = ((TransportReplicationAction.ConcreteShardRequest)requestOrWrappedRequest).getRequest(); + } + return (R) requestOrWrappedRequest; + } + private static ThreadPool threadPool; private ClusterService clusterService; diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java index 210379dad99b4..2d2a4cec7e3b0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java @@ -102,6 +102,15 @@ public void run() { ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); + + assertThrows(client().admin().indices().prepareAnalyze("test", "this is a test"), + ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE + ); + + assertThrows(client().admin().indices().prepareAnalyze("no_index", "this is a test"), + ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE + ); + assertThrows(client().prepareSearch("test").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE ); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java similarity index 79% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java rename to core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java index 7fe1dae698c31..4f9b13d91cee6 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java @@ -16,22 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.analysis.common; +package org.elasticsearch.indices.analyze; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequestBuilder; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.hamcrest.core.IsNull; import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.Map; -import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -40,19 +37,13 @@ import static org.hamcrest.Matchers.startsWith; -public class AnalyzeActionTests extends ESIntegTestCase { - @Override - protected Collection> nodePlugins() { - return singletonList(CommonAnalysisPlugin.class); - } - +public class AnalyzeActionIT extends ESIntegTestCase { public void testSimpleAnalyzerTests() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); for (int i = 0; i < 10; i++) { - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze(indexOrAlias(), "this is a test").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "this is a test").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("this")); @@ -82,32 +73,25 @@ public void testSimpleAnalyzerTests() throws Exception { } public void testAnalyzeNumericField() throws IOException { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) - .addMapping("test", "long", "type=long", "double", "type=double")); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "long", "type=long", "double", "type=double")); ensureGreen("test"); - expectThrows(IllegalArgumentException.class, () -> client().admin().indices() - .prepareAnalyze(indexOrAlias(), "123").setField("long").get()); + expectThrows(IllegalArgumentException.class, + () -> client().admin().indices().prepareAnalyze(indexOrAlias(), "123").setField("long").get()); - expectThrows(IllegalArgumentException.class, () -> client().admin().indices() - .prepareAnalyze(indexOrAlias(), "123.0").setField("double").get()); + expectThrows(IllegalArgumentException.class, + () -> client().admin().indices().prepareAnalyze(indexOrAlias(), "123.0").setField("double").get()); } public void testAnalyzeWithNoIndex() throws Exception { - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setTokenizer("keyword") - .addTokenFilter("lowercase").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setTokenizer("standard") - .addTokenFilter("lowercase") - .addTokenFilter("reverse").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("standard").addTokenFilter("lowercase").addTokenFilter("reverse").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("siht")); @@ -118,9 +102,7 @@ public void testAnalyzeWithNoIndex() throws Exception { token = analyzeResponse.getTokens().get(3); assertThat(token.getTerm(), equalTo("tset")); - analyzeResponse = client().admin().indices().prepareAnalyze("of course") - .setTokenizer("standard") - .addTokenFilter("stop").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("of course").setTokenizer("standard").addTokenFilter("stop").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("course")); assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1)); @@ -133,40 +115,23 @@ public void testAnalyzeWithCharFilters() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.builder().put(indexSettings()) .put("index.analysis.char_filter.custom_mapping.type", "mapping") - .putArray("index.analysis.char_filter.custom_mapping.mappings", - "ph=>f", "qu=>q") - .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", - "standard") - .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", - "custom_mapping"))); + .putArray("index.analysis.char_filter.custom_mapping.mappings", "ph=>f", "qu=>q") + .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard") + .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "custom_mapping"))); ensureGreen(); - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("

THIS IS A

TEST") - .setTokenizer("standard") - .addCharFilter("html_strip").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("

THIS IS A

TEST").setTokenizer("standard").addCharFilter("html_strip").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(4)); - analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setTokenizer("keyword") - .addTokenFilter("lowercase") - .addCharFilter("html_strip").get(); + analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("html_strip").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test")); - analyzeResponse = client().admin().indices() - .prepareAnalyze(indexOrAlias(), "jeff quit phish") - .setTokenizer("keyword") - .addTokenFilter("lowercase") - .addCharFilter("custom_mapping").get(); + analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "jeff quit phish").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("custom_mapping").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("jeff qit fish")); - analyzeResponse = client().admin().indices() - .prepareAnalyze(indexOrAlias(), "jeff quit fish") - .setTokenizer("standard") - .addCharFilter("html_strip") - .addCharFilter("custom_mapping").get(); + analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "jeff quit fish").setTokenizer("standard").addCharFilter("html_strip").addCharFilter("custom_mapping").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(3)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("jeff")); @@ -185,10 +150,7 @@ public void testAnalyzeWithNonDefaultPostionLength() throws Exception { .putArray("index.analysis.analyzer.custom_syns.filter", "lowercase", "syns"))); ensureGreen(); - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("say what the fudge") - .setIndex("test") - .setAnalyzer("custom_syns").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("say what the fudge").setIndex("test").setAnalyzer("custom_syns").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(5)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); @@ -235,8 +197,7 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception { .setType("document").setSource("simple", "type=text,analyzer=simple").get(); for (int i = 0; i < 10; i++) { - final AnalyzeRequestBuilder requestBuilder = client().admin().indices() - .prepareAnalyze("THIS IS A TEST"); + final AnalyzeRequestBuilder requestBuilder = client().admin().indices().prepareAnalyze("THIS IS A TEST"); requestBuilder.setIndex(indexOrAlias()); requestBuilder.setField("document.simple"); AnalyzeResponse analyzeResponse = requestBuilder.get(); @@ -251,12 +212,10 @@ public void testAnalyzerWithFieldOrTypeTests() throws Exception { // issue #5974 public void testThatStandardAndDefaultAnalyzersAreSame() throws Exception { - AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test") - .setAnalyzer("standard").get(); + AnalyzeResponse response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("standard").get(); assertTokens(response, "this", "is", "a", "test"); - response = client().admin().indices().prepareAnalyze("this is a test") - .setAnalyzer("default").get(); + response = client().admin().indices().prepareAnalyze("this is a test").setAnalyzer("default").get(); assertTokens(response, "this", "is", "a", "test"); response = client().admin().indices().prepareAnalyze("this is a test").get(); @@ -278,8 +237,8 @@ public void testAnalyzerWithMultiValues() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); - client().admin().indices().preparePutMapping("test").setType("document") - .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + client().admin().indices().preparePutMapping("test") + .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"}; @@ -318,50 +277,36 @@ public void testDetailAnalyze() throws Exception { ensureGreen(); for (int i = 0; i < 10; i++) { - AnalyzeResponse analyzeResponse = admin().indices().prepareAnalyze() - .setIndex(indexOrAlias()).setText("THIS IS A PHISH") - .setExplain(true) - .addCharFilter("my_mapping") - .setTokenizer("keyword") - .addTokenFilter("lowercase").get(); + AnalyzeResponse analyzeResponse = admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText("THIS IS A PHISH") + .setExplain(true).addCharFilter("my_mapping").setTokenizer("keyword").addTokenFilter("lowercase").get(); assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], - equalTo("THIS IS A FISH")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("THIS IS A FISH")); //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), - equalTo("THIS IS A FISH")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), - equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), - equalTo(15)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("THIS IS A FISH")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15)); //tokenfilters assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), - equalTo("this is a fish")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), - equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), - equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), - equalTo(15)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("this is a fish")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(15)); } } public void testDetailAnalyzeWithNoIndex() throws Exception { //analyzer only - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("THIS IS A TEST") - .setExplain(true) - .setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setExplain(true).setAnalyzer("simple").get(); assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); @@ -372,10 +317,8 @@ public void testDetailAnalyzeWithNoIndex() throws Exception { public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { //analyzer only - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("THIS IS A TEST") - .setExplain(true) - .setAnalyzer("simple").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") + .setExplain(true).setAnalyzer("simple").get(); assertThat(analyzeResponse.detail().tokenizer(), IsNull.nullValue()); assertThat(analyzeResponse.detail().tokenfilters(), IsNull.nullValue()); @@ -385,28 +328,23 @@ public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { //custom analyzer analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST") - .setExplain(true) - .addCharFilter("html_strip") - .setTokenizer("keyword") - .addTokenFilter("lowercase").get(); + .setExplain(true).addCharFilter("html_strip").setTokenizer("keyword").addTokenFilter("lowercase").get(); assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("html_strip")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], - equalTo("\nTHIS IS A TEST\n")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("\nTHIS IS A TEST\n")); //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), - equalTo("\nTHIS IS A TEST\n")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("\nTHIS IS A TEST\n")); //tokenfilters assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), - equalTo("\nthis is a test\n")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("\nthis is a test\n")); + //check other attributes analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") @@ -415,44 +353,35 @@ public void testDetailAnalyzeCustomAnalyzerWithNoIndex() throws Exception { assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), - equalTo("troubl")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); String[] expectedAttributesKey = { "bytes", "positionLength", "keyword"}; - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), - equalTo(expectedAttributesKey.length)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); Object extendedAttribute; for (String key : expectedAttributesKey) { - extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2] - .getAttributes().get(key); + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); assertThat(extendedAttribute, notNullValue()); } } public void testDetailAnalyzeSpecifyAttributes() throws Exception { - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze("This is troubled") - .setExplain(true) - .setTokenizer("standard") - .addTokenFilter("snowball") - .setAttributes("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("This is troubled") + .setExplain(true).setTokenizer("standard").addTokenFilter("snowball").setAttributes("keyword").get(); assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("snowball")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), - equalTo("troubl")); - String[] expectedAttributesKey = {"keyword"}; - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), - equalTo(expectedAttributesKey.length)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("troubl")); + String[] expectedAttributesKey = { + "keyword"}; + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().size(), equalTo(expectedAttributesKey.length)); Object extendedAttribute; for (String key : expectedAttributesKey) { - extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2] - .getAttributes().get(key); + extendedAttribute = analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getAttributes().get(key); assertThat(extendedAttribute, notNullValue()); } } @@ -461,17 +390,11 @@ public void testDetailAnalyzeWithMultiValues() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); ensureGreen(); client().admin().indices().preparePutMapping("test") - .setType("document") - .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"}; - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze() - .setIndex(indexOrAlias()) - .setText(texts) - .setExplain(true) - .setField("simple") - .setText(texts).get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) + .setExplain(true).setField("simple").setText(texts).execute().get(); assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple")); assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7)); @@ -501,30 +424,23 @@ public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exceptio .put("index.analysis.analyzer.test_analyzer.position_increment_gap", "100") .put("index.analysis.analyzer.test_analyzer.tokenizer", "standard") .putArray("index.analysis.analyzer.test_analyzer.char_filter", "my_mapping") - .putArray("index.analysis.analyzer.test_analyzer.filter", - "snowball", "lowercase"))); + .putArray("index.analysis.analyzer.test_analyzer.filter", "snowball", "lowercase"))); ensureGreen(); client().admin().indices().preparePutMapping("test") - .setType("document") - .setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); + .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get(); //only analyzer = String[] texts = new String[]{"this is a PHISH", "the troubled text"}; - AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze() - .setIndex(indexOrAlias()) - .setText(texts) - .setExplain(true) - .setAnalyzer("test_analyzer").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts) + .setExplain(true).setAnalyzer("test_analyzer").setText(texts).execute().get(); // charfilter assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("my_mapping")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(2)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], - equalTo("this is a FISH")); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[1], - equalTo("the troubled text")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("this is a FISH")); + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[1], equalTo("the troubled text")); // tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("standard")); @@ -609,81 +525,54 @@ public void testCustomTokenFilterInRequest() throws Exception { assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("whitespace")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("Foo")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), - equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("buzz")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), - equalTo(4)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(4)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(8)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("test")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), - equalTo(9)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(9)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(13)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1)); // tokenfilter(lowercase) assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2)); assertThat(analyzeResponse.detail().tokenfilters()[0].getName(), equalTo("lowercase")); assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens().length, equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), - equalTo("foo")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), - equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), - equalTo(3)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), - equalTo(0)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), - equalTo(1)); - - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), - equalTo("buzz")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), - equalTo(4)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), - equalTo(8)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), - equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), - equalTo(1)); - - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), - equalTo("test")); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), - equalTo(9)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), - equalTo(13)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), - equalTo(2)); - assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getTerm(), equalTo("foo")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(3)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), equalTo(1)); + + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), equalTo("buzz")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), equalTo(4)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), equalTo(8)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), equalTo(1)); + + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("test")); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), equalTo(9)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), equalTo(13)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), equalTo(2)); + assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), equalTo(1)); // tokenfilter({"type": "stop", "stopwords": ["foo", "buzz"]}) - assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), - equalTo("_anonymous_tokenfilter_[1]")); + assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("_anonymous_tokenfilter_[1]")); assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getTerm(), - equalTo("test")); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), - equalTo(9)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), - equalTo(13)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), - equalTo(2)); - assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getTerm(), equalTo("test")); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), equalTo(9)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), equalTo(13)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), equalTo(2)); + assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), equalTo(1)); } @@ -702,21 +591,16 @@ public void testCustomCharFilterInRequest() throws Exception { assertThat(analyzeResponse.detail().analyzer(), IsNull.nullValue()); //charfilters assertThat(analyzeResponse.detail().charfilters().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getName(), - equalTo("_anonymous_charfilter_[0]")); + assertThat(analyzeResponse.detail().charfilters()[0].getName(), equalTo("_anonymous_charfilter_[0]")); assertThat(analyzeResponse.detail().charfilters()[0].getTexts().length, equalTo(1)); - assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], - equalTo("jeff qit fish")); - // tokenizer + assertThat(analyzeResponse.detail().charfilters()[0].getTexts()[0], equalTo("jeff qit fish")); + //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("keyword")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), - equalTo("jeff qit fish")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), - equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("jeff qit fish")); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); } @@ -733,42 +617,33 @@ public void testCustomTokenizerInRequest() throws Exception { .setExplain(true) .get(); - // tokenizer + //tokenizer assertThat(analyzeResponse.detail().tokenizer().getName(), equalTo("_anonymous_tokenizer")); assertThat(analyzeResponse.detail().tokenizer().getTokens().length, equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("go")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), - equalTo(0)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(2)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("oo")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(3)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("od")); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), - equalTo(2)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(2)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(4)); assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2)); - assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), - equalTo(1)); + assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1)); } public void testAnalyzeKeywordField() throws IOException { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) - .addMapping("test", "keyword", "type=keyword")); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "keyword", "type=keyword")); ensureGreen("test"); - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze(indexOrAlias(), "ABC") - .setField("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("ABC")); @@ -786,9 +661,7 @@ public void testAnalyzeNormalizedKeywordField() throws IOException { .addMapping("test", "keyword", "type=keyword,normalizer=my_normalizer")); ensureGreen("test"); - AnalyzeResponse analyzeResponse = client().admin().indices() - .prepareAnalyze(indexOrAlias(), "ABC") - .setField("keyword").get(); + AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get(); assertThat(analyzeResponse.getTokens().size(), equalTo(1)); AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0); assertThat(token.getTerm(), equalTo("abc")); @@ -796,5 +669,6 @@ public void testAnalyzeNormalizedKeywordField() throws IOException { assertThat(token.getEndOffset(), equalTo(3)); assertThat(token.getPosition(), equalTo(0)); assertThat(token.getPositionLength(), equalTo(1)); + } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java similarity index 88% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java rename to core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java index 5a0c0dc602e10..958b9e5222fad 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/RestAnalyzeActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.analysis.common; +package org.elasticsearch.rest.action.admin.indices; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.common.bytes.BytesArray; @@ -91,32 +91,27 @@ public void testParseXContentForAnalyzeRequestWithCustomFilters() throws Excepti assertThat(analyzeRequest.charFilters().get(0).definition, notNullValue()); } - public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() - throws Exception { - RestAnalyzeAction action = new RestAnalyzeAction( - Settings.EMPTY, mock(RestController.class)); + public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception { + RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class)); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> action.handleRequest(request, null, null)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null)); assertThat(e.getMessage(), equalTo("Failed to parse request body")); } - public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() - throws Exception { + public void testParseXContentForAnalyzeRequestWithUnknownParamThrowsException() throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); XContentParser invalidContent = createParser(XContentFactory.jsonBuilder() .startObject() .field("text", "THIS IS A TEST") .field("unknown", "keyword") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest)); assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]")); } - public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() - throws Exception { + public void testParseXContentForAnalyzeRequestWithInvalidStringExplainParamThrowsException() throws Exception { AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test"); XContentParser invalidExplain = createParser(XContentFactory.jsonBuilder() .startObject() @@ -135,8 +130,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, + new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [filters]")); } @@ -147,8 +142,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("token_filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, + new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [token_filters]")); } @@ -159,8 +154,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("char_filters", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, + new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [char_filters]")); } @@ -171,8 +166,8 @@ public void testDeprecatedParamIn2xException() throws Exception { .field("tokenizer", "keyword") .array("token_filter", "lowercase") .endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - RestAnalyzeAction.buildFromContent(parser, new AnalyzeRequest("for test"))); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser, + new AnalyzeRequest("for test"))); assertThat(e.getMessage(), startsWith("Unknown parameter [token_filter]")); } } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index da511e4d5326b..bfd1bbdcc97b8 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -19,31 +19,15 @@ package org.elasticsearch.analysis.common; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.function.Supplier; -import static java.util.Collections.singletonList; - -public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ActionPlugin { +public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { @Override public Map> getTokenFilters() { Map> filters = new HashMap<>(); @@ -52,18 +36,4 @@ public Map> getTokenFilters() { filters.put("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new); return filters; } - - @Override - public List> getActions() { - return singletonList( - new ActionHandler<>(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class)); - } - - @Override - public List getRestHandlers(Settings settings, RestController restController, - ClusterSettings clusterSettings, IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster) { - return singletonList(new RestAnalyzeAction(settings, restController)); - } } diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java deleted file mode 100644 index f0b9d48bace60..0000000000000 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/AnalyzeRequestTests.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.analysis.common; - -import org.elasticsearch.action.IndicesRequestTestCase; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Tests routing the analyze request. - */ -public class AnalyzeRequestTests extends IndicesRequestTestCase { - @Override - protected Collection> nodePlugins() { - List> plugins = new ArrayList<>(); - plugins.addAll(super.nodePlugins()); - plugins.add(CommonAnalysisPlugin.class); - return plugins; - } - - public void testAnalyze() { - String analyzeShardAction = AnalyzeAction.NAME + "[s]"; - interceptTransportActions(analyzeShardAction); - - AnalyzeRequest analyzeRequest = new AnalyzeRequest(randomIndexOrAlias()); - analyzeRequest.text("text"); - internalCluster().coordOnlyNodeClient().admin().indices() - .analyze(analyzeRequest).actionGet(); - - clearInterceptedActions(); - assertSameIndices(analyzeRequest, analyzeShardAction); - } -} diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java index acef0c426399f..c7d0abceed3db 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -153,6 +154,7 @@ public void testFillsAttributes() throws IOException { } public void testWithIndexAnalyzers() throws IOException { + AnalyzeRequest request = new AnalyzeRequest(); request.analyzer("standard"); request.text("the quick brown fox"); diff --git a/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java b/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java deleted file mode 100644 index cd43fcaeffe93..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/action/IndicesRequestTestCase.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action; - -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.support.replication.ReplicationRequest; -import org.elasticsearch.action.support.replication.TransportReplicationAction; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.plugins.NetworkPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.transport.TransportChannel; -import org.elasticsearch.transport.TransportInterceptor; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; -import org.junit.After; -import org.junit.Before; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static java.util.Collections.singletonList; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.emptyIterable; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.hasItem; - -public class IndicesRequestTestCase extends ESIntegTestCase { - private final List indices = new ArrayList<>(); - - @Override - protected Collection> nodePlugins() { - return singletonList(InterceptingTransportService.TestPlugin.class); - } - - @Before - public void setup() { - int numIndices = iterations(1, 5); - for (int i = 0; i < numIndices; i++) { - indices.add("test" + i); - } - for (String index : indices) { - assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias"))); - } - ensureGreen(); - } - - @After - public void cleanUp() { - assertAllRequestsHaveBeenConsumed(); - indices.clear(); - } - - protected String randomIndexOrAlias() { - String index = randomFrom(indices); - if (randomBoolean()) { - return index + "-alias"; - } else { - return index; - } - } - - protected String[] randomIndicesOrAliases() { - int count = randomIntBetween(1, indices.size() * 2); //every index has an alias - String[] indices = new String[count]; - for (int i = 0; i < count; i++) { - indices[i] = randomIndexOrAlias(); - } - return indices; - } - - protected String[] randomUniqueIndicesOrAliases() { - Set uniqueIndices = new HashSet<>(); - int count = randomIntBetween(1, this.indices.size()); - while (uniqueIndices.size() < count) { - uniqueIndices.add(randomFrom(this.indices)); - } - String[] indices = new String[count]; - int i = 0; - for (String index : uniqueIndices) { - indices[i++] = randomBoolean() ? index + "-alias" : index; - } - return indices; - } - - protected static void assertSameIndices(IndicesRequest originalRequest, String... actions) { - assertSameIndices(originalRequest, false, actions); - } - - protected static void assertSameIndicesOptionalRequests(IndicesRequest originalRequest, - String... actions) { - assertSameIndices(originalRequest, true, actions); - } - - protected static void assertSameIndices(IndicesRequest originalRequest, boolean optional, - String... actions) { - for (String action : actions) { - List requests = consumeTransportRequests(action); - if (!optional) { - assertThat("no internal requests intercepted for action [" + action + "]", - requests.size(), greaterThan(0)); - } - for (TransportRequest internalRequest : requests) { - IndicesRequest indicesRequest = convertRequest(internalRequest); - assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), - equalTo(originalRequest.indices())); - assertThat(indicesRequest.indicesOptions(), - equalTo(originalRequest.indicesOptions())); - } - } - } - protected static void assertIndicesSubset(List indices, String... actions) { - //indices returned by each bulk shard request need to be a subset of the original indices - for (String action : actions) { - List requests = consumeTransportRequests(action); - assertThat("no internal requests intercepted for action [" + action + "]", - requests.size(), greaterThan(0)); - for (TransportRequest internalRequest : requests) { - IndicesRequest indicesRequest = convertRequest(internalRequest); - for (String index : indicesRequest.indices()) { - assertThat(indices, hasItem(index)); - } - } - } - } - private static IndicesRequest convertRequest(TransportRequest request) { - final IndicesRequest indicesRequest; - if (request instanceof IndicesRequest) { - indicesRequest = (IndicesRequest) request; - } else { - indicesRequest = resolveRequest(request); - } - return indicesRequest; - } - private static > R resolveRequest( - TransportRequest requestOrWrappedRequest) { - if (requestOrWrappedRequest instanceof TransportReplicationAction.ConcreteShardRequest) { - requestOrWrappedRequest = - ((TransportReplicationAction.ConcreteShardRequest) requestOrWrappedRequest) - .getRequest(); - } - return (R) requestOrWrappedRequest; - } - - protected static void assertAllRequestsHaveBeenConsumed() { - withInterceptingTransportService(its -> - assertThat(its.requests.entrySet(), emptyIterable())); - } - - protected static void clearInterceptedActions() { - withInterceptingTransportService(InterceptingTransportService::clearInterceptedActions); - } - - protected static void interceptTransportActions(String... actions) { - withInterceptingTransportService(its -> its.interceptTransportActions(actions)); - } - - protected static List consumeTransportRequests(String action) { - List requests = new ArrayList<>(); - withInterceptingTransportService(its -> { - List transportRequests = its.consumeRequests(action); - if (transportRequests != null) { - requests.addAll(transportRequests); - } - }); - return requests; - } - - private static void withInterceptingTransportService( - Consumer callback) { - for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { - InterceptingTransportService its = pluginsService - .filterPlugins(InterceptingTransportService.TestPlugin.class) - .stream().findFirst().get().instance; - callback.accept(its); - } - - } - - private static class InterceptingTransportService implements TransportInterceptor { - public static class TestPlugin extends Plugin implements NetworkPlugin { - public final InterceptingTransportService instance = new InterceptingTransportService(); - @Override - public List getTransportInterceptors( - NamedWriteableRegistry namedWriteableRegistry, ThreadContext threadContext) { - return Collections.singletonList(instance); - } - } - - private final Set actions = new HashSet<>(); - - private final Map> requests = new HashMap<>(); - - @Override - public TransportRequestHandler interceptHandler( - String action, String executor, boolean forceExecution, - TransportRequestHandler actualHandler) { - return new InterceptingRequestHandler<>(action, actualHandler); - } - - synchronized List consumeRequests(String action) { - return requests.remove(action); - } - - synchronized void interceptTransportActions(String... actions) { - Collections.addAll(this.actions, actions); - } - - synchronized void clearInterceptedActions() { - actions.clear(); - } - - - private class InterceptingRequestHandler - implements TransportRequestHandler { - private final TransportRequestHandler requestHandler; - private final String action; - - InterceptingRequestHandler(String action, TransportRequestHandler requestHandler) { - this.requestHandler = requestHandler; - this.action = action; - } - - @Override - public void messageReceived(T request, TransportChannel channel, Task task) - throws Exception { - synchronized (InterceptingTransportService.this) { - if (actions.contains(action)) { - List requestList = requests.get(action); - if (requestList == null) { - requestList = new ArrayList<>(); - requestList.add(request); - requests.put(action, requestList); - } else { - requestList.add(request); - } - } - } - requestHandler.messageReceived(request, channel, task); - } - - @Override - public void messageReceived(T request, TransportChannel channel) throws Exception { - messageReceived(request, channel, null); - } - } - } -} From 5d45b69651cf0839ee028704367e46892b1e02c5 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 17:12:46 -0400 Subject: [PATCH 22/28] Break out tests --- .../test/analysis-common/20_analyzers.yaml | 11 ++ .../test/analysis-common/30_tokenizers.yaml | 27 ++++ .../analysis-common/40_token_filters.yaml | 11 ++ .../test/analysis-common/50_char_filters.yaml | 13 ++ .../test/indices.analyze/10_analyze.yaml | 124 ------------------ .../test/indices.analyze/10_analyze.yaml | 113 ++++++++++++++++ 6 files changed, 175 insertions(+), 124 deletions(-) create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml delete mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml new file mode 100644 index 0000000000000..d5dee9fb31037 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml @@ -0,0 +1,11 @@ +## Smoke tests for analyzers included in the analysis-common module + +"whitespace": + - do: + indices.analyze: + body: + text: Foo Bar! + analyzer: whitespace + - length: { tokens: 2 } + - match: { tokens.0.token: Foo } + - match: { tokens.1.token: Bar! } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml new file mode 100644 index 0000000000000..2ed330ad84354 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml @@ -0,0 +1,27 @@ +## Smoke tests for tokenizers included in the analysis-common module + +"keyword": + - do: + indices.analyze: + body: + text: Foo Bar! + tokenizer: keyword + - length: { tokens: 2 } + - match: { tokens.0.token: Foo Bar! } + +--- +"nGram": + - do: + indices.analyze: + body: + text: good, + explain: true + tokenizer: + type: nGram, + min_gram: 2, + max_gram: 2 + - length: { detail.tokenizer.tokens: 3 } + - match: { detail.tokenizer.name: _anonymous_tokenizer } + - match: { detail.tokenizer.tokens.0.token: go } + - match: { detail.tokenizer.tokens.1.token: oo } + - match: { detail.tokenizer.tokens.2.token: od } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml new file mode 100644 index 0000000000000..a0e1234c0ac41 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml @@ -0,0 +1,11 @@ +## Smoke tests for token filters included in the analysis-common module + +"lowercase": + - do: + indices.analyze: + body: + text: Foo Bar! + tokenizer: keyword + filter: [lowercase] + - length: { tokens: 2 } + - match: { tokens.0.token: foo bar! } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml new file mode 100644 index 0000000000000..92894e38489a1 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml @@ -0,0 +1,13 @@ +## Smoke tests for analyzers included in the analysis-common module + +"mapping": + - do: + indices.analyze: + body: + text: jeff quit phish, + tokenzier: keyword + char_filter: + - type: mapping, + mappings: ["ph => f", "qu => q"] + - length: { tokens: 1 } + - match: { tokens.0.token: "jeff qit fish" } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml deleted file mode 100644 index e4997381556e2..0000000000000 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# The rest tests for the indices.analyze action live in the common-analysis -# module because they depend on many analyzers not in core. - -"Basic test": - - do: - indices.analyze: - body: - text: Foo Bar - - length: { tokens: 2 } - - match: { tokens.0.token: foo } - - match: { tokens.1.token: bar } - ---- -"Tokenizer and filter": - - do: - indices.analyze: - body: - filter: [lowercase] - text: Foo Bar - tokenizer: keyword - - length: { tokens: 1 } - - match: { tokens.0.token: foo bar } - ---- -"Index and field": - - do: - indices.create: - index: test - body: - mappings: - test: - properties: - text: - type: text - analyzer: whitespace - - - do: - indices.analyze: - index: test - body: - field: text - text: Foo Bar! - - length: { tokens: 2 } - - match: { tokens.0.token: Foo } - - match: { tokens.1.token: Bar! } ---- -"JSON in Body": - - do: - indices.analyze: - body: { "text": "Foo Bar", "filter": ["lowercase"], "tokenizer": keyword } - - length: {tokens: 1 } - - match: { tokens.0.token: foo bar } ---- -"Array text": - - do: - indices.analyze: - body: { "text": ["Foo Bar", "Baz"], "filter": ["lowercase"], "tokenizer": keyword } - - length: {tokens: 2 } - - match: { tokens.0.token: foo bar } - - match: { tokens.1.token: baz } ---- -"Detail response with Analyzer": - - do: - indices.analyze: - body: {"text": "This is troubled", "analyzer": standard, "explain": "true"} - - length: { detail.analyzer.tokens: 3 } - - match: { detail.analyzer.name: standard } - - match: { detail.analyzer.tokens.0.token: this } - - match: { detail.analyzer.tokens.1.token: is } - - match: { detail.analyzer.tokens.2.token: troubled } ---- -"Detail output spcified attribute": - - do: - indices.analyze: - body: {"text": "This is troubled", "char_filter": ["html_strip"], "filter": ["snowball"], "tokenizer": standard, "explain": true, "attributes": ["keyword"]} - - length: { detail.charfilters: 1 } - - length: { detail.tokenizer.tokens: 3 } - - length: { detail.tokenfilters.0.tokens: 3 } - - match: { detail.tokenizer.name: standard } - - match: { detail.tokenizer.tokens.0.token: This } - - match: { detail.tokenizer.tokens.1.token: is } - - match: { detail.tokenizer.tokens.2.token: troubled } - - match: { detail.tokenfilters.0.name: snowball } - - match: { detail.tokenfilters.0.tokens.0.token: This } - - match: { detail.tokenfilters.0.tokens.1.token: is } - - match: { detail.tokenfilters.0.tokens.2.token: troubl } - - match: { detail.tokenfilters.0.tokens.2.keyword: false } - ---- -"Custom filter in request": - - do: - indices.analyze: - body: { "text": "Foo Bar Buzz", "filter": ["lowercase", { "type": "stop", "stopwords": ["foo", "buzz"]}], "tokenizer": whitespace, "explain": true } - - length: {detail.tokenizer.tokens: 3 } - - length: {detail.tokenfilters.0.tokens: 3 } - - length: {detail.tokenfilters.1.tokens: 1 } - - match: { detail.tokenizer.name: whitespace } - - match: { detail.tokenizer.tokens.0.token: Foo } - - match: { detail.tokenizer.tokens.1.token: Bar } - - match: { detail.tokenizer.tokens.2.token: Buzz } - - match: { detail.tokenfilters.0.name: lowercase } - - match: { detail.tokenfilters.0.tokens.0.token: foo } - - match: { detail.tokenfilters.0.tokens.1.token: bar } - - match: { detail.tokenfilters.0.tokens.2.token: buzz } - - match: { detail.tokenfilters.1.name: "_anonymous_tokenfilter_[1]" } - - match: { detail.tokenfilters.1.tokens.0.token: bar } ---- -"Custom char_filter in request": - - do: - indices.analyze: - body: { "text": "jeff quit phish", "char_filter": [{"type": "mapping", "mappings": ["ph => f", "qu => q"]}], "tokenizer": keyword } - - length: {tokens: 1 } - - match: { tokens.0.token: "jeff qit fish" } - ---- -"Custom tokenizer in request": - - do: - indices.analyze: - body: { "text": "good", "tokenizer": {"type": "nGram", "min_gram": 2, "max_gram": 2}, "explain": true } - - length: {detail.tokenizer.tokens: 3 } - - match: { detail.tokenizer.name: _anonymous_tokenizer } - - match: { detail.tokenizer.tokens.0.token: go } - - match: { detail.tokenizer.tokens.1.token: oo } - - match: { detail.tokenizer.tokens.2.token: od } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml new file mode 100644 index 0000000000000..3dad544c4eb8f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml @@ -0,0 +1,113 @@ +"Basic test": + - do: + indices.analyze: + body: + text: Foo Bar + - length: { tokens: 2 } + - match: { tokens.0.token: foo } + - match: { tokens.1.token: bar } + +--- +"Index and field": + - do: + indices.create: + index: test + body: + mappings: + test: + properties: + text: + type: text + analyzer: standard + + - do: + indices.analyze: + index: test + body: + field: text + text: Foo Bar! + - length: { tokens: 2 } + - match: { tokens.0.token: Foo } + - match: { tokens.1.token: Bar } + +--- +"Array text": + - do: + indices.analyze: + body: + text: ["Foo Bar", "Baz"] + tokenizer: standard + - length: {tokens: 2 } + - match: { tokens.0.token: foo } + - match: { tokens.1.token: bar } + - match: { tokens.1.token: baz } + +--- +"Detail response with Analyzer": + - do: + indices.analyze: + body: + text: This is troubled + analyzer: standard + explain: true + - length: { detail.analyzer.tokens: 3 } + - match: { detail.analyzer.name: standard } + - match: { detail.analyzer.tokens.0.token: this } + - match: { detail.analyzer.tokens.1.token: is } + - match: { detail.analyzer.tokens.2.token: troubled } + +--- +"Detail response with attribute": + - do: + indices.analyze: + body: + text: This is troubled + analyzer: standard + explain: true + attributes: ["keyword"] + - length: { detail.analyzer.tokens: 3 } + - match: { detail.analyzer.name: standard } + - match: { detail.analyzer.tokens.0.token: this } + - match: { detail.analyzer.tokens.0.keyword: false } + - match: { detail.analyzer.tokens.1.token: is } + - match: { detail.analyzer.tokens.1.keyword: false } + - match: { detail.analyzer.tokens.2.token: troubled } + - match: { detail.analyzer.tokens.2.keyword: false } + +--- +"Detail output spcified attribute": + - do: + indices.analyze: + body: {"text": "This is troubled", "char_filter": ["html_strip"], "filter": ["snowball"], "tokenizer": standard, "explain": true, "attributes": ["keyword"]} + - length: { detail.charfilters: 1 } + - length: { detail.tokenizer.tokens: 3 } + - length: { detail.tokenfilters.0.tokens: 3 } + - match: { detail.tokenizer.name: standard } + - match: { detail.tokenizer.tokens.0.token: This } + - match: { detail.tokenizer.tokens.1.token: is } + - match: { detail.tokenizer.tokens.2.token: troubled } + - match: { detail.tokenfilters.0.name: snowball } + - match: { detail.tokenfilters.0.tokens.0.token: This } + - match: { detail.tokenfilters.0.tokens.1.token: is } + - match: { detail.tokenfilters.0.tokens.2.token: troubl } + - match: { detail.tokenfilters.0.tokens.2.keyword: false } + +--- +"Custom filter in request": + - do: + indices.analyze: + body: + text: Foo Bar Buzz + tokenizer: standard + explain: true + filter: + - type: stop + stopwords: ["foo", "buzz"] + - length: {detail.tokenizer.tokens: 3 } + - length: {detail.tokenfilters.0.tokens: 1 } + - match: { detail.tokenizer.name: standard } + - match: { detail.tokenizer.tokens.0.token: Foo } + - match: { detail.tokenizer.tokens.1.token: Bar } + - match: { detail.tokenizer.tokens.2.token: Buzz } + - match: { detail.tokenfilters.0.name: "_anonymous_tokenfilter_[1]" } + - match: { detail.tokenfilters.0.tokens.0.token: bar } From fced8124a62df275ba3f834d92cc2cea4605a6ad Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 17:23:14 -0400 Subject: [PATCH 23/28] Add smoke tests for moved analyzers --- .../test/analysis-common/20_analyzers.yaml | 6 +- .../analysis-common/40_token_filters.yaml | 73 ++++++++++++++++++- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml index d5dee9fb31037..9fb34e7a821c6 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yaml @@ -6,6 +6,6 @@ body: text: Foo Bar! analyzer: whitespace - - length: { tokens: 2 } - - match: { tokens.0.token: Foo } - - match: { tokens.1.token: Bar! } + - length: { tokens: 2 } + - match: { tokens.0.token: Foo } + - match: { tokens.1.token: Bar! } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml index a0e1234c0ac41..bf6cd7e54b69f 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml @@ -1,5 +1,16 @@ ## Smoke tests for token filters included in the analysis-common module +"asciifolding": + - do: + indices.analyze: + body: + text: Musée d'Orsay + tokenizer: keyword + filter: [asciifolding] + - length: { tokens: 1 } + - match: { tokens.0.token: Musee d'Orsay } + +--- "lowercase": - do: indices.analyze: @@ -7,5 +18,63 @@ text: Foo Bar! tokenizer: keyword filter: [lowercase] - - length: { tokens: 2 } - - match: { tokens.0.token: foo bar! } + - length: { tokens: 1 } + - match: { tokens.0.token: foo bar! } + +--- +"word_delimiter": + - do: + indices.analyze: + body: + text: the qu1ck brown fox + tokenizer: standard + filter: [word_delimiter] + - length: { tokens: 5 } + - match: { tokens.0.token: the } + - match: { tokens.1.token: qu } + - match: { tokens.2.token: ck } + - match: { tokens.3.token: brown } + - match: { tokens.4.token: fox } + + - do: + indices.analyze: + body: + text: the qu1ck brown fox + tokenizer: standard + filter: + - type: word_delimiter + split_on_numeric: false + - length: { tokens: 4 } + - match: { tokens.0.token: the } + - match: { tokens.1.token: qu1ck } + - match: { tokens.2.token: brown } + - match: { tokens.3.token: fox } + +--- +"word_delimiter_graph": + - do: + indices.analyze: + body: + text: the qu1ck brown fox + tokenizer: standard + filter: [word_delimiter_graph] + - length: { tokens: 5 } + - match: { tokens.0.token: the } + - match: { tokens.1.token: qu } + - match: { tokens.2.token: ck } + - match: { tokens.3.token: brown } + - match: { tokens.4.token: fox } + + - do: + indices.analyze: + body: + text: the qu1ck brown fox + tokenizer: standard + filter: + - type: word_delimiter_graph + split_on_numeric: false + - length: { tokens: 4 } + - match: { tokens.0.token: the } + - match: { tokens.1.token: qu1ck } + - match: { tokens.2.token: brown } + - match: { tokens.3.token: fox } From eaafe409bec1ccd1b7cb6ed06a090d7d4c056f5e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Mar 2017 17:40:22 -0400 Subject: [PATCH 24/28] Cleanup tests --- .../test/analysis-common/30_tokenizers.yaml | 8 +-- .../analysis-common/40_token_filters.yaml | 22 ++++--- .../test/analysis-common/50_char_filters.yaml | 6 +- .../test/indices.analyze/10_analyze.yaml | 62 ++++--------------- 4 files changed, 32 insertions(+), 66 deletions(-) diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml index 2ed330ad84354..174a15f772bd9 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yaml @@ -6,7 +6,7 @@ body: text: Foo Bar! tokenizer: keyword - - length: { tokens: 2 } + - length: { tokens: 1 } - match: { tokens.0.token: Foo Bar! } --- @@ -14,11 +14,11 @@ - do: indices.analyze: body: - text: good, + text: good explain: true tokenizer: - type: nGram, - min_gram: 2, + type: nGram + min_gram: 2 max_gram: 2 - length: { detail.tokenizer.tokens: 3 } - match: { detail.tokenizer.name: _anonymous_tokenizer } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml index bf6cd7e54b69f..ac5bcb82e5783 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yaml @@ -29,12 +29,13 @@ text: the qu1ck brown fox tokenizer: standard filter: [word_delimiter] - - length: { tokens: 5 } + - length: { tokens: 6 } - match: { tokens.0.token: the } - match: { tokens.1.token: qu } - - match: { tokens.2.token: ck } - - match: { tokens.3.token: brown } - - match: { tokens.4.token: fox } + - match: { tokens.2.token: "1" } + - match: { tokens.3.token: ck } + - match: { tokens.4.token: brown } + - match: { tokens.5.token: fox } - do: indices.analyze: @@ -43,7 +44,7 @@ tokenizer: standard filter: - type: word_delimiter - split_on_numeric: false + split_on_numerics: false - length: { tokens: 4 } - match: { tokens.0.token: the } - match: { tokens.1.token: qu1ck } @@ -58,12 +59,13 @@ text: the qu1ck brown fox tokenizer: standard filter: [word_delimiter_graph] - - length: { tokens: 5 } + - length: { tokens: 6 } - match: { tokens.0.token: the } - match: { tokens.1.token: qu } - - match: { tokens.2.token: ck } - - match: { tokens.3.token: brown } - - match: { tokens.4.token: fox } + - match: { tokens.2.token: "1" } + - match: { tokens.3.token: ck } + - match: { tokens.4.token: brown } + - match: { tokens.5.token: fox } - do: indices.analyze: @@ -72,7 +74,7 @@ tokenizer: standard filter: - type: word_delimiter_graph - split_on_numeric: false + split_on_numerics: false - length: { tokens: 4 } - match: { tokens.0.token: the } - match: { tokens.1.token: qu1ck } diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml index 92894e38489a1..06775a2a72290 100644 --- a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yaml @@ -4,10 +4,10 @@ - do: indices.analyze: body: - text: jeff quit phish, - tokenzier: keyword + text: jeff quit phish + tokenizer: keyword char_filter: - - type: mapping, + - type: mapping mappings: ["ph => f", "qu => q"] - length: { tokens: 1 } - match: { tokens.0.token: "jeff qit fish" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml index 3dad544c4eb8f..93ce5c8c80734 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.analyze/10_analyze.yaml @@ -27,8 +27,8 @@ field: text text: Foo Bar! - length: { tokens: 2 } - - match: { tokens.0.token: Foo } - - match: { tokens.1.token: Bar } + - match: { tokens.0.token: foo } + - match: { tokens.1.token: bar } --- "Array text": @@ -37,10 +37,10 @@ body: text: ["Foo Bar", "Baz"] tokenizer: standard - - length: {tokens: 2 } - - match: { tokens.0.token: foo } - - match: { tokens.1.token: bar } - - match: { tokens.1.token: baz } + - length: { tokens: 3 } + - match: { tokens.0.token: Foo } + - match: { tokens.1.token: Bar } + - match: { tokens.2.token: Baz } --- "Detail response with Analyzer": @@ -56,58 +56,22 @@ - match: { detail.analyzer.tokens.1.token: is } - match: { detail.analyzer.tokens.2.token: troubled } ---- -"Detail response with attribute": - - do: - indices.analyze: - body: - text: This is troubled - analyzer: standard - explain: true - attributes: ["keyword"] - - length: { detail.analyzer.tokens: 3 } - - match: { detail.analyzer.name: standard } - - match: { detail.analyzer.tokens.0.token: this } - - match: { detail.analyzer.tokens.0.keyword: false } - - match: { detail.analyzer.tokens.1.token: is } - - match: { detail.analyzer.tokens.1.keyword: false } - - match: { detail.analyzer.tokens.2.token: troubled } - - match: { detail.analyzer.tokens.2.keyword: false } - ---- -"Detail output spcified attribute": - - do: - indices.analyze: - body: {"text": "This is troubled", "char_filter": ["html_strip"], "filter": ["snowball"], "tokenizer": standard, "explain": true, "attributes": ["keyword"]} - - length: { detail.charfilters: 1 } - - length: { detail.tokenizer.tokens: 3 } - - length: { detail.tokenfilters.0.tokens: 3 } - - match: { detail.tokenizer.name: standard } - - match: { detail.tokenizer.tokens.0.token: This } - - match: { detail.tokenizer.tokens.1.token: is } - - match: { detail.tokenizer.tokens.2.token: troubled } - - match: { detail.tokenfilters.0.name: snowball } - - match: { detail.tokenfilters.0.tokens.0.token: This } - - match: { detail.tokenfilters.0.tokens.1.token: is } - - match: { detail.tokenfilters.0.tokens.2.token: troubl } - - match: { detail.tokenfilters.0.tokens.2.keyword: false } - --- "Custom filter in request": - do: indices.analyze: body: - text: Foo Bar Buzz + text: foo bar buzz tokenizer: standard explain: true filter: - type: stop stopwords: ["foo", "buzz"] - - length: {detail.tokenizer.tokens: 3 } - - length: {detail.tokenfilters.0.tokens: 1 } + - length: { detail.tokenizer.tokens: 3 } + - length: { detail.tokenfilters.0.tokens: 1 } - match: { detail.tokenizer.name: standard } - - match: { detail.tokenizer.tokens.0.token: Foo } - - match: { detail.tokenizer.tokens.1.token: Bar } - - match: { detail.tokenizer.tokens.2.token: Buzz } - - match: { detail.tokenfilters.0.name: "_anonymous_tokenfilter_[1]" } + - match: { detail.tokenizer.tokens.0.token: foo } + - match: { detail.tokenizer.tokens.1.token: bar } + - match: { detail.tokenizer.tokens.2.token: buzz } + - match: { detail.tokenfilters.0.name: "_anonymous_tokenfilter_[0]" } - match: { detail.tokenfilters.0.tokens.0.token: bar } From d4bf67cd4c874d219a9907f7fe829f385841bd24 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Sat, 1 Apr 2017 12:53:19 -0400 Subject: [PATCH 25/28] Remove now duplicate tests --- .../common/TransportAnalyzeActionTests.java | 291 ------------------ 1 file changed, 291 deletions(-) delete mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java deleted file mode 100644 index c7d0abceed3db..0000000000000 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/TransportAnalyzeActionTests.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.analysis.common; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.action.admin.indices.analyze.TransportAnalyzeAction; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.AnalysisRegistry; -import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.mapper.AllFieldMapper; -import org.elasticsearch.indices.analysis.AnalysisModule; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.IndexSettingsModule; - -import java.io.IOException; -import java.util.List; - -import static java.util.Collections.singletonList; - -/** - * More "intense" version of a unit test with the same name that is in core. This one has access to - * the analyzers in this module. - */ -public class TransportAnalyzeActionTests extends ESTestCase { - private IndexAnalyzers indexAnalyzers; - private AnalysisRegistry registry; - private Environment environment; - - @Override - public void setUp() throws Exception { - super.setUp(); - Settings settings = Settings.builder().put( - Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); - - Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) - .put("index.analysis.filter.wordDelimiter.type", "word_delimiter") - .put("index.analysis.filter.wordDelimiter.split_on_numerics", false) - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", - "lowercase", "wordDelimiter") - .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.custom_analyzer.filter", - "lowercase", "wordDelimiter") - .put("index.analysis.tokenizer.trigram.type", "ngram") - .put("index.analysis.tokenizer.trigram.min_gram", 3) - .put("index.analysis.tokenizer.trigram.max_gram", 3) - .put("index.analysis.filter.synonym.type", "synonym") - .putArray("index.analysis.filter.synonym.synonyms", "kimchy => shay") - .put("index.analysis.filter.synonym.tokenizer", "trigram") - .put("index.analysis.filter.synonym.min_gram", 3) - .put("index.analysis.filter.synonym.max_gram", 3).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - environment = new Environment(settings); - registry = new AnalysisModule(environment, singletonList(new CommonAnalysisPlugin())) - .getAnalysisRegistry(); - indexAnalyzers = registry.build(idxSettings); - } - - public void testNoIndexAnalyzers() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); - request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - null, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - - request.analyzer(null); - request.tokenizer("whitespace"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("word_delimiter"); - request.text("the qu1ck brown fox"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - randomBoolean() ? indexAnalyzers : null, registry, environment); - tokens = analyze.getTokens(); - assertEquals(6, tokens.size()); - assertEquals("qu", tokens.get(1).getTerm()); - assertEquals("1", tokens.get(2).getTerm()); - assertEquals("ck", tokens.get(3).getTerm()); - - request.analyzer(null); - request.tokenizer("whitespace"); - request.addCharFilter("html_strip"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("word_delimiter"); - request.text("

the qu1ck brown fox

"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - randomBoolean() ? indexAnalyzers : null, registry, environment); - tokens = analyze.getTokens(); - assertEquals(6, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals("qu", tokens.get(1).getTerm()); - assertEquals("1", tokens.get(2).getTerm()); - assertEquals("ck", tokens.get(3).getTerm()); - assertEquals("brown", tokens.get(4).getTerm()); - assertEquals("fox", tokens.get(5).getTerm()); - } - - public void testFillsAttributes() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); - request.text("the 1 brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - null, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals(0, tokens.get(0).getStartOffset()); - assertEquals(3, tokens.get(0).getEndOffset()); - assertEquals(0, tokens.get(0).getPosition()); - assertEquals("", tokens.get(0).getType()); - - assertEquals("1", tokens.get(1).getTerm()); - assertEquals(4, tokens.get(1).getStartOffset()); - assertEquals(5, tokens.get(1).getEndOffset()); - assertEquals(1, tokens.get(1).getPosition()); - assertEquals("", tokens.get(1).getType()); - - assertEquals("brown", tokens.get(2).getTerm()); - assertEquals(6, tokens.get(2).getStartOffset()); - assertEquals(11, tokens.get(2).getEndOffset()); - assertEquals(2, tokens.get(2).getPosition()); - assertEquals("", tokens.get(2).getType()); - - assertEquals("fox", tokens.get(3).getTerm()); - assertEquals(12, tokens.get(3).getStartOffset()); - assertEquals(15, tokens.get(3).getEndOffset()); - assertEquals(3, tokens.get(3).getPosition()); - assertEquals("", tokens.get(3).getType()); - } - - public void testWithIndexAnalyzers() throws IOException { - - AnalyzeRequest request = new AnalyzeRequest(); - request.analyzer("standard"); - request.text("the quick brown fox"); - request.analyzer("custom_analyzer"); - request.text("the qu1ck brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - indexAnalyzers, registry, environment); - List tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - - request.analyzer("whitespace"); - request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, - registry, environment); - tokens = analyze.getTokens(); - assertEquals(4, tokens.size()); - - request.analyzer("custom_analyzer"); - request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, - registry, environment); - tokens = analyze.getTokens(); - assertEquals(5, tokens.size()); - - request.analyzer(null); - request.tokenizer("whitespace"); - request.addTokenFilter("lowercase"); - request.addTokenFilter("wordDelimiter"); - request.text("the qu1ck brown fox-dog"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, - registry, environment); - tokens = analyze.getTokens(); - assertEquals(5, tokens.size()); - assertEquals("the", tokens.get(0).getTerm()); - assertEquals("qu1ck", tokens.get(1).getTerm()); - assertEquals("brown", tokens.get(2).getTerm()); - assertEquals("fox", tokens.get(3).getTerm()); - assertEquals("dog", tokens.get(4).getTerm()); - - request.analyzer(null); - request.tokenizer("trigram"); - request.addTokenFilter("synonym"); - request.text("kimchy"); - analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, - registry, environment); - tokens = analyze.getTokens(); - assertEquals(2, tokens.size()); - assertEquals("sha", tokens.get(0).getTerm()); - assertEquals("hay", tokens.get(1).getTerm()); - } - - public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .analyzer("custom_analyzer") - .text("the qu1ck brown fox-dog"), - AllFieldMapper.NAME, null, null, registry, environment)); - assertEquals(e.getMessage(), "failed to find global analyzer [custom_analyzer]"); - } - - public void testUnknown() throws IOException { - boolean notGlobal = randomBoolean(); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .analyzer("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, - registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find analyzer [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global analyzer [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, - registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find tokenizer under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global tokenizer under [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("whitespace") - .addTokenFilter("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, - registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find token filter under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global token filter under [foobar]"); - } - - e = expectThrows(IllegalArgumentException.class, - () -> TransportAnalyzeAction.analyze( - new AnalyzeRequest() - .tokenizer("whitespace") - .addTokenFilter("lowercase") - .addCharFilter("foobar") - .text("the qu1ck brown fox"), - AllFieldMapper.NAME, null, notGlobal ? indexAnalyzers : null, - registry, environment)); - if (notGlobal) { - assertEquals(e.getMessage(), "failed to find char filter under [foobar]"); - } else { - assertEquals(e.getMessage(), "failed to find global char filter under [foobar]"); - } - } - - public void testNonPreBuildTokenFilter() throws IOException { - AnalyzeRequest request = new AnalyzeRequest(); - request.tokenizer("whitespace"); - request.addTokenFilter("min_hash"); - request.text("the quick brown fox"); - AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, - indexAnalyzers, registry, environment); - List tokens = analyze.getTokens(); - int default_hash_count = 1; - int default_bucket_size = 512; - int default_hash_set_size = 1; - assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, - tokens.size()); - } -} From dfdc53c5debd0b526904a33ca8300946d69552c8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 12 Apr 2017 15:00:24 -0400 Subject: [PATCH 26/28] Cleanup after merge --- .../elasticsearch/index/analysis/AnalysisRegistryTests.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index 5e3ac01bf8107..efe17de2e8fee 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -40,14 +40,9 @@ import org.elasticsearch.test.VersionUtils; import java.io.IOException; -<<<<<<< HEAD import java.util.Collections; import java.util.HashMap; import java.util.Map; -======= -import java.util.ArrayList; -import java.util.List; ->>>>>>> master import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; From 51609e191ee8b3a3957c24f231a79cfc106372e4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 12 Apr 2017 15:07:50 -0400 Subject: [PATCH 27/28] Remove test accidentally added in merge --- .../index/analysis/AnalysisRegistryTests.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java index efe17de2e8fee..0edd2fbe2c08f 100644 --- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java +++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java @@ -116,22 +116,6 @@ public void testOverrideDefaultSearchAnalyzer() { assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); } - public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() { - Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), - VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1)); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - Map> analyzers = new HashMap<>(); - analyzers.put("default_index", analyzerProvider("default_index")); - analyzers.put("default_search", analyzerProvider("default_search")); - IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings), - analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); - assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); - assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); - assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class)); - assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " + - "instead for index [index]"); - } - /** * Tests that {@code camelCase} filter names and {@code snake_case} filter names don't collide. */ From fac537ee4f2cf8c4a1c4c32b42fe0034e5a6c939 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 19 Apr 2017 17:05:08 -0400 Subject: [PATCH 28/28] Catch exception --- .../analysis/common/CommonAnalysisClientYamlTestSuiteIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java index c77966e81b0f5..b5d6bf23b3eda 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java @@ -24,15 +24,13 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import java.io.IOException; - public class CommonAnalysisClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public CommonAnalysisClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate testCandidate) { super(testCandidate); } @ParametersFactory - public static Iterable parameters() throws IOException { + public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } }