elastic · martijnvg · Jul 17, 2017 · Jul 13, 2017
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -31,15 +31,12 @@
 import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.ApostropheFilterFactory;
 import org.elasticsearch.index.analysis.ArabicAnalyzerProvider;
-import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.ArmenianAnalyzerProvider;
 import org.elasticsearch.index.analysis.BasqueAnalyzerProvider;
 import org.elasticsearch.index.analysis.BrazilianAnalyzerProvider;
 import org.elasticsearch.index.analysis.BrazilianStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.BulgarianAnalyzerProvider;
-import org.elasticsearch.index.analysis.CJKBigramFilterFactory;
-import org.elasticsearch.index.analysis.CJKWidthFilterFactory;
 import org.elasticsearch.index.analysis.CatalanAnalyzerProvider;
 import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.ChineseAnalyzerProvider;
@@ -62,14 +59,11 @@
 import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.GalicianAnalyzerProvider;
 import org.elasticsearch.index.analysis.GermanAnalyzerProvider;
-import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.GreekAnalyzerProvider;
 import org.elasticsearch.index.analysis.HindiAnalyzerProvider;
-import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.HungarianAnalyzerProvider;
 import org.elasticsearch.index.analysis.HunspellTokenFilterFactory;
-import org.elasticsearch.index.analysis.IndicNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.IndonesianAnalyzerProvider;
 import org.elasticsearch.index.analysis.IrishAnalyzerProvider;
 import org.elasticsearch.index.analysis.ItalianAnalyzerProvider;
@@ -88,7 +82,6 @@
 import org.elasticsearch.index.analysis.PatternAnalyzerProvider;
 import org.elasticsearch.index.analysis.PatternTokenizerFactory;
 import org.elasticsearch.index.analysis.PersianAnalyzerProvider;
-import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider;
 import org.elasticsearch.index.analysis.PreConfiguredCharFilter;
 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
@@ -97,13 +90,10 @@
 import org.elasticsearch.index.analysis.RussianAnalyzerProvider;
 import org.elasticsearch.index.analysis.RussianStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory;
-import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory;
-import org.elasticsearch.index.analysis.SerbianNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.analysis.SimpleAnalyzerProvider;
 import org.elasticsearch.index.analysis.SnowballAnalyzerProvider;
 import org.elasticsearch.index.analysis.SoraniAnalyzerProvider;
-import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.SpanishAnalyzerProvider;
 import org.elasticsearch.index.analysis.StandardAnalyzerProvider;
 import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzerProvider;
@@ -202,20 +192,10 @@ private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(Li
         tokenFilters.register("french_stem", FrenchStemTokenFilterFactory::new);
         tokenFilters.register("german_stem", GermanStemTokenFilterFactory::new);
         tokenFilters.register("russian_stem", RussianStemTokenFilterFactory::new);
-        tokenFilters.register("arabic_normalization", ArabicNormalizationFilterFactory::new);
-        tokenFilters.register("german_normalization", GermanNormalizationFilterFactory::new);
-        tokenFilters.register("hindi_normalization", HindiNormalizationFilterFactory::new);
-        tokenFilters.register("indic_normalization", IndicNormalizationFilterFactory::new);
-        tokenFilters.register("sorani_normalization", SoraniNormalizationFilterFactory::new);
-        tokenFilters.register("persian_normalization", PersianNormalizationFilterFactory::new);
-        tokenFilters.register("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
         tokenFilters.register("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
-        tokenFilters.register("serbian_normalization", SerbianNormalizationFilterFactory::new);
 
         tokenFilters.register("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory
             (indexSettings, name, settings, hunspellService)));
-        tokenFilters.register("cjk_bigram", CJKBigramFilterFactory::new);
-        tokenFilters.register("cjk_width", CJKWidthFilterFactory::new);
 
         tokenFilters.register("apostrophe", ApostropheFilterFactory::new);
         tokenFilters.register("classic", ClassicFilterFactory::new);

diff --git a/...sis/ArabicNormalizationFilterFactory.java → ...mon/ArabicNormalizationFilterFactory.java b/...sis/ArabicNormalizationFilterFactory.java → ...mon/ArabicNormalizationFilterFactory.java
@@ -16,17 +16,19 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public ArabicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    ArabicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...ndex/analysis/CJKBigramFilterFactory.java → ...alysis/common/CJKBigramFilterFactory.java b/...ndex/analysis/CJKBigramFilterFactory.java → ...alysis/common/CJKBigramFilterFactory.java
@@ -17,14 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cjk.CJKBigramFilter;
 import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -49,7 +50,7 @@ public final class CJKBigramFilterFactory extends AbstractTokenFilterFactory {
     private final int flags;
     private final boolean outputUnigrams;
 
-    public CJKBigramFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    CJKBigramFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
         outputUnigrams = settings.getAsBooleanLenientForPreEs6Indices(
             indexSettings.getIndexVersionCreated(), "output_unigrams", false, deprecationLogger);

diff --git a/...index/analysis/CJKWidthFilterFactory.java → ...nalysis/common/CJKWidthFilterFactory.java b/...index/analysis/CJKWidthFilterFactory.java → ...nalysis/common/CJKWidthFilterFactory.java
@@ -17,17 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.cjk.CJKWidthFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
+    CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/...analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java
@@ -71,6 +71,7 @@
 import org.elasticsearch.index.analysis.PreConfiguredCharFilter;
 import org.elasticsearch.index.analysis.PreConfiguredTokenFilter;
 import org.elasticsearch.index.analysis.PreConfiguredTokenizer;
+import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
@@ -118,6 +119,16 @@ public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
         filters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
         filters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
         filters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
+        filters.put("arabic_normalization", ArabicNormalizationFilterFactory::new);
+        filters.put("german_normalization", GermanNormalizationFilterFactory::new);
+        filters.put("hindi_normalization", HindiNormalizationFilterFactory::new);
+        filters.put("indic_normalization", IndicNormalizationFilterFactory::new);
+        filters.put("persian_normalization", PersianNormalizationFilterFactory::new);
+        filters.put("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
+        filters.put("serbian_normalization", SerbianNormalizationFilterFactory::new);
+        filters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
+        filters.put("cjk_width", CJKWidthFilterFactory::new);
+        filters.put("cjk_bigram", CJKBigramFilterFactory::new);
         return filters;
     }
 

diff --git a/...sis/GermanNormalizationFilterFactory.java → ...mon/GermanNormalizationFilterFactory.java b/...sis/GermanNormalizationFilterFactory.java → ...mon/GermanNormalizationFilterFactory.java
@@ -16,20 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.de.GermanNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 /**
  * Factory for {@link GermanNormalizationFilter}
  */
 public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public GermanNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    GermanNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...ysis/HindiNormalizationFilterFactory.java → ...mmon/HindiNormalizationFilterFactory.java b/...ysis/HindiNormalizationFilterFactory.java → ...mmon/HindiNormalizationFilterFactory.java
@@ -16,20 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.hi.HindiNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 /**
  * Factory for {@link HindiNormalizationFilter}
  */
 public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public HindiNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    HindiNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...ysis/IndicNormalizationFilterFactory.java → ...mmon/IndicNormalizationFilterFactory.java b/...ysis/IndicNormalizationFilterFactory.java → ...mmon/IndicNormalizationFilterFactory.java
@@ -16,20 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.in.IndicNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 /**
  * Factory for {@link IndicNormalizationFilter}
  */
 public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public IndicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    IndicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...is/PersianNormalizationFilterFactory.java → ...on/PersianNormalizationFilterFactory.java b/...is/PersianNormalizationFilterFactory.java → ...on/PersianNormalizationFilterFactory.java
@@ -16,17 +16,19 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.fa.PersianNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public PersianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    PersianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...andinavianNormalizationFilterFactory.java → ...andinavianNormalizationFilterFactory.java b/...andinavianNormalizationFilterFactory.java → ...andinavianNormalizationFilterFactory.java
@@ -16,20 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.miscellaneous.ScandinavianNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 /**
  * Factory for {@link ScandinavianNormalizationFilter}
  */
 public class ScandinavianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public ScandinavianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    ScandinavianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...is/SerbianNormalizationFilterFactory.java → ...on/SerbianNormalizationFilterFactory.java b/...is/SerbianNormalizationFilterFactory.java → ...on/SerbianNormalizationFilterFactory.java
@@ -17,17 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.sr.SerbianNormalizationFilter;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
 
 public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
-    public SerbianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+    SerbianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
     }
 

diff --git a/...index/analysis/CJKFilterFactoryTests.java → ...nalysis/common/CJKFilterFactoryTests.java b/...index/analysis/CJKFilterFactoryTests.java → ...nalysis/common/CJKFilterFactoryTests.java
@@ -17,23 +17,32 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.analysis.common;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.miscellaneous.DisableGraphAttribute;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.io.StringReader;
 
 public class CJKFilterFactoryTests extends ESTokenStreamTestCase {
-    private static final String RESOURCE = "/org/elasticsearch/index/analysis/cjk_analysis.json";
+    private static final String RESOURCE = "/org/elasticsearch/analysis/common/cjk_analysis.json";
+
+    private ESTestCase.TestAnalysis analysis;
+
+    @Before
+    public void setup() throws IOException {
+        analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE, new CommonAnalysisPlugin());
+    }
 
     public void testDefault() throws IOException {
-        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_bigram");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" };
@@ -43,7 +52,6 @@ public void testDefault() throws IOException {
     }
 
     public void testNoFlags() throws IOException {
-        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_no_flags");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" };
@@ -53,7 +61,6 @@ public void testNoFlags() throws IOException {
     }
 
     public void testHanOnly() throws IOException {
-        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_han_only");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多", "く", "の",  "学生", "が",  "試験", "に",  "落", "ち", "た"  };
@@ -63,7 +70,6 @@ public void testHanOnly() throws IOException {
     }
 
     public void testHanUnigramOnly() throws IOException {
-        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
         TokenFilterFactory tokenFilter = analysis.tokenFilter.get("cjk_han_unigram_only");
         String source = "多くの学生が試験に落ちた。";
         String[] expected = new String[]{"多", "く", "の",  "学", "学生", "生", "が",  "試", "試験", "験", "に",  "落", "ち", "た"  };
@@ -73,7 +79,6 @@ public void testHanUnigramOnly() throws IOException {
     }
 
     public void testDisableGraph() throws IOException {
-        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromClassPath(createTempDir(), RESOURCE);
         TokenFilterFactory allFlagsFactory = analysis.tokenFilter.get("cjk_all_flags");
         TokenFilterFactory hanOnlyFactory = analysis.tokenFilter.get("cjk_han_only");