Start building analysis-common module

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -1096,7 +1096,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergeSchedulerConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]VersionType.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]ASCIIFoldingTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AbstractCharFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AbstractIndexAnalyzerProvider.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AbstractTokenFilterFactory.java" checks="LineLength" />
@@ -1225,8 +1224,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]UpperCaseTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WhitespaceAnalyzerProvider.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WhitespaceTokenizerFactory.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WordDelimiterGraphTokenFilterFactory.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WordDelimiterTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]compound[/\\]AbstractCompoundWordTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]compound[/\\]DictionaryCompoundWordTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]compound[/\\]HyphenationCompoundWordTokenFilterFactory.java" checks="LineLength" />
@@ -2686,11 +2683,8 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SettingsListenerIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]VersionTypeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]WaitUntilRefreshIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]ASCIIFoldingTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisRegistryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisTestsHelper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]BaseWordDelimiterTokenFilterFactoryTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CJKFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CharFilterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CompoundAnalysisTests.java" checks="LineLength" />
@@ -2709,8 +2703,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StemmerTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StopAnalyzerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StopTokenFilterTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WordDelimiterGraphTokenFilterFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]WordDelimiterTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]commongrams[/\\]CommonGramsTokenFilterFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]filter1[/\\]MyFilterTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]synonyms[/\\]SynonymsAnalysisTests.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/index/analysis/EdgeNGramTokenFilterFactory.java

@@ -71,4 +71,9 @@ public TokenStream create(TokenStream tokenStream) {
 
         return result;
     }
+
+    @Override
+    public boolean breaksFastVectorHighlighter() {
+        return true;
+    }
 }

core/src/main/java/org/elasticsearch/index/analysis/TokenFilterFactory.java

@@ -20,10 +20,20 @@
 package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter;
 
 public interface TokenFilterFactory {
-
     String name();
 
     TokenStream create(TokenStream tokenStream);
+
+    /**
+     * Does this analyzer mess up the {@link OffsetAttribute}s in such as way as to break the
+     * {@link FastVectorHighlighter}? If this is {@code true} then the
+     * {@linkplain FastVectorHighlighter} will attempt to work around the broken offsets.
+     */
+    default boolean breaksFastVectorHighlighter() {
+        return false;
+    }
 }

core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java

@@ -25,7 +25,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.ApostropheFilterFactory;
@@ -140,8 +139,6 @@
 import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory;
 import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider;
 import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
-import org.elasticsearch.index.analysis.WordDelimiterGraphTokenFilterFactory;
-import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
 import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
 import org.elasticsearch.plugins.AnalysisPlugin;
@@ -205,7 +202,6 @@ private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(Li
         NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = new NamedRegistry<>("token_filter");
         tokenFilters.register("stop", StopTokenFilterFactory::new);
         tokenFilters.register("reverse", ReverseTokenFilterFactory::new);
-        tokenFilters.register("asciifolding", ASCIIFoldingTokenFilterFactory::new);
         tokenFilters.register("length", LengthTokenFilterFactory::new);
         tokenFilters.register("lowercase", LowerCaseTokenFilterFactory::new);
         tokenFilters.register("uppercase", UpperCaseTokenFilterFactory::new);
@@ -225,8 +221,6 @@ private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(Li
         tokenFilters.register("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
         tokenFilters.register("snowball", SnowballTokenFilterFactory::new);
         tokenFilters.register("stemmer", StemmerTokenFilterFactory::new);
-        tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new);
-        tokenFilters.register("word_delimiter_graph", WordDelimiterGraphTokenFilterFactory::new);
         tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
         tokenFilters.register("elision", ElisionTokenFilterFactory::new);
         tokenFilters.register("flatten_graph", FlattenGraphTokenFilterFactory::new);

core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java

@@ -26,15 +26,9 @@
 import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
 import org.apache.lucene.search.vectorhighlight.FragmentsBuilder;
 import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.Version;
 import org.elasticsearch.index.analysis.CustomAnalyzer;
-import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
-import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
-import org.elasticsearch.index.analysis.NGramTokenFilterFactory;
-import org.elasticsearch.index.analysis.NGramTokenizerFactory;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
-import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
 import org.elasticsearch.index.mapper.FieldMapper;
 
 import java.util.Comparator;
@@ -56,7 +50,7 @@ private FragmentBuilderHelper() {
     public static WeightedFragInfo fixWeightedFragInfo(FieldMapper mapper, Field[] values, WeightedFragInfo fragInfo) {
         assert fragInfo != null : "FragInfo must not be null";
         assert mapper.fieldType().name().equals(values[0].name()) : "Expected FieldMapper for field " + values[0].name();
-        if (!fragInfo.getSubInfos().isEmpty() && (containsBrokenAnalysis(mapper.fieldType().indexAnalyzer()))) {
+        if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(mapper.fieldType().indexAnalyzer())) {
             /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time
              * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort
              * the fragments based on their offsets rather than using soley the positions as it is done in
@@ -91,8 +85,7 @@ private static boolean containsBrokenAnalysis(Analyzer analyzer) {
             final CustomAnalyzer a = (CustomAnalyzer) analyzer;
             TokenFilterFactory[] tokenFilters = a.tokenFilters();
             for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
-                if (tokenFilterFactory instanceof WordDelimiterTokenFilterFactory
-                        || tokenFilterFactory instanceof EdgeNGramTokenFilterFactory) {
+                if (tokenFilterFactory.breaksFastVectorHighlighter()) {
                     return true;
                 }
             }

core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java

@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.action.admin.indices;
 
+import org.apache.lucene.analysis.MockTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
@@ -27,18 +29,28 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
-import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 
+/**
+ * Tests for {@link TransportAnalyzeAction}. See the more "intense" version of this test in the
+ * {@code common-analysis} module.
+ */
 public class TransportAnalyzeActionTests extends ESTestCase {
 
     private IndexAnalyzers indexAnalyzers;
@@ -53,23 +65,28 @@ public void setUp() throws Exception {
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
-                .put("index.analysis.filter.wordDelimiter.type", "word_delimiter")
-                .put("index.analysis.filter.wordDelimiter.split_on_numerics", false)
-                .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace")
-                .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")
-                .put("index.analysis.analyzer.custom_analyzer.tokenizer", "whitespace")
-                .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter")
-                .put("index.analysis.tokenizer.trigram.type", "ngram")
-                .put("index.analysis.tokenizer.trigram.min_gram", 3)
-                .put("index.analysis.tokenizer.trigram.max_gram", 3)
-                .put("index.analysis.filter.synonym.type", "synonym")
-                .putArray("index.analysis.filter.synonym.synonyms", "kimchy => shay")
-                .put("index.analysis.filter.synonym.tokenizer", "trigram")
-                .put("index.analysis.filter.synonym.min_gram", 3)
-                .put("index.analysis.filter.synonym.max_gram", 3).build();
+                .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
+                .put("index.analysis.analyzer.custom_analyzer.filter", "mock").build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
         environment = new Environment(settings);
-        registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry();
+        AnalysisPlugin plugin = new AnalysisPlugin() {
+            class MockFactory extends AbstractTokenFilterFactory {
+                MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+                    super(indexSettings, name, settings);
+                }
+
+                @Override
+                public TokenStream create(TokenStream tokenStream) {
+                    return new MockTokenFilter(tokenStream, MockTokenFilter.ENGLISH_STOPSET);
+                }
+            }
+
+            @Override
+            public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+                return singletonMap("mock", MockFactory::new);
+            }
+        };
+        registry = new AnalysisModule(environment, singletonList(plugin)).getAnalysisRegistry();
         indexAnalyzers = registry.build(idxSettings);
     }
 
@@ -143,51 +160,44 @@ public void testFillsAttributes() throws IOException {
     }
 
     public void testWithIndexAnalyzers() throws IOException {
-
         AnalyzeRequest request = new AnalyzeRequest();
-        request.analyzer("standard");
         request.text("the quick brown fox");
         request.analyzer("custom_analyzer");
-        request.text("the qu1ck brown fox");
         AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
-        assertEquals(4, tokens.size());
+        assertEquals(3, tokens.size());
+        assertEquals("quick", tokens.get(0).getTerm());
+        assertEquals("brown", tokens.get(1).getTerm());
+        assertEquals("fox", tokens.get(2).getTerm());
 
-        request.analyzer("whitespace");
-        request.text("the qu1ck brown fox-dog");
+        request.analyzer("standard");
         analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
         assertEquals(4, tokens.size());
+        assertEquals("the", tokens.get(0).getTerm());
+        assertEquals("quick", tokens.get(1).getTerm());
+        assertEquals("brown", tokens.get(2).getTerm());
+        assertEquals("fox", tokens.get(3).getTerm());
 
-        request.analyzer("custom_analyzer");
-        request.text("the qu1ck brown fox-dog");
-        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
-        tokens = analyze.getTokens();
-        assertEquals(5, tokens.size());
-
+        // Switch the analyzer out for just a tokenizer
         request.analyzer(null);
-        request.tokenizer("whitespace");
-        request.addTokenFilter("lowercase");
-        request.addTokenFilter("wordDelimiter");
-        request.text("the qu1ck brown fox-dog");
+        request.tokenizer("standard");
         analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
-        assertEquals(5, tokens.size());
+        assertEquals(4, tokens.size());
         assertEquals("the", tokens.get(0).getTerm());
-        assertEquals("qu1ck", tokens.get(1).getTerm());
+        assertEquals("quick", tokens.get(1).getTerm());
         assertEquals("brown", tokens.get(2).getTerm());
         assertEquals("fox", tokens.get(3).getTerm());
-        assertEquals("dog", tokens.get(4).getTerm());
 
-        request.analyzer(null);
-        request.tokenizer("trigram");
-        request.addTokenFilter("synonym");
-        request.text("kimchy");
+        // Now try applying our token filter
+        request.addTokenFilter("mock");
         analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
         tokens = analyze.getTokens();
-        assertEquals(2, tokens.size());
-        assertEquals("sha", tokens.get(0).getTerm());
-        assertEquals("hay", tokens.get(1).getTerm());
+        assertEquals(3, tokens.size());
+        assertEquals("quick", tokens.get(0).getTerm());
+        assertEquals("brown", tokens.get(1).getTerm());
+        assertEquals("fox", tokens.get(2).getTerm());
     }
 
     public void testGetIndexAnalyserWithoutIndexAnalyzers() throws IOException {

core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java

@@ -22,5 +22,5 @@
 import org.elasticsearch.AnalysisFactoryTestCase;
 
 public class AnalysisFactoryTests extends AnalysisFactoryTestCase {
-    // tests are inherited
+    // tests are inherited and nothing needs to be defined here
 }