From 9d9b6af14d5c3f9ff4cdcc72b3f488e2d2c563c8 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:51:08 +0200 Subject: [PATCH 1/5] unmute tests --- muted-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index ec47ec92bbac6..3e104e8a5b0bf 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -163,12 +163,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 -- class: org.elasticsearch.upgrades.QueryBuilderBWCIT - method: testQueryBuilderBWC {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116990 -- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT - method: testQueryBuilderBWC {p0=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/116989 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 From 02ddf8205360b818ba18beba96dc457a6843e70e Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Fri, 3 Jan 2025 14:53:03 +0200 Subject: [PATCH 2/5] revert --- muted-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3e104e8a5b0bf..ec47ec92bbac6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -163,6 +163,12 @@ tests: - class: org.elasticsearch.xpack.ml.integration.RegressionIT method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet issue: https://github.com/elastic/elasticsearch/issues/117805 +- class: org.elasticsearch.upgrades.QueryBuilderBWCIT + method: testQueryBuilderBWC {cluster=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116990 +- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT + method: testQueryBuilderBWC {p0=UPGRADED} + issue: https://github.com/elastic/elasticsearch/issues/116989 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT method: testEsqlRcs2UnavailableRemoteScenarios issue: https://github.com/elastic/elasticsearch/issues/117419 From 61bed253fc98802436243ca745ba5bde4f68ac56 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Tue, 21 Jan 2025 17:12:59 +0200 Subject: [PATCH 3/5] Revert tests --- .../common/EdgeNGramTokenizerTests.java | 59 +++++++++++++-- ...DelimiterGraphTokenFilterFactoryTests.java | 71 ++++++++++++++----- 2 files changed, 109 insertions(+), 21 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java index c998e927e25a8..27ff2548c126e 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java @@ -18,12 +18,14 @@ import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.test.ESTokenStreamTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.io.StringReader; @@ -49,12 +51,61 @@ private IndexAnalyzers buildAnalyzers(IndexVersion version, String tokenizer) th } public void testPreConfiguredTokenizer() throws IOException { - try (IndexAnalyzers indexAnalyzers = buildAnalyzers(IndexVersion.current(), "edge_ngram")) { - NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); - assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" }); + // Before 7.3 we return ngrams of length 1 only + { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0) + ); + try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) { + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "test", new String[]{"t"}); + } } + // Check deprecated name as well + { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0) + ); + try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) { + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "test", new String[]{"t"}); + } + } + + // Afterwards, we return ngrams of length 1 and 2, to match the default factory settings + { + try (IndexAnalyzers indexAnalyzers = buildAnalyzers(IndexVersion.current(), "edge_ngram")) { + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"}); + } + } + + // Check deprecated name as well, needs version before 8.0 because throws IAE after that + { + try ( + IndexAnalyzers indexAnalyzers = buildAnalyzers( + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_3_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ), + "edgeNGram" + ) + ) { + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"}); + + } + } } public void testCustomTokenChars() throws IOException { diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java index 4995fe844c9c5..c739e517df91b 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.IndexService.IndexCreationContext; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalysisTestsHelper; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -25,6 +26,7 @@ import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.io.StringReader; @@ -182,26 +184,61 @@ public void testIgnoreKeywords() throws IOException { } public void testPreconfiguredFilter() throws IOException { - Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); - Settings indexSettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard") - .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph") - .build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); + // Before 7.3 we don't adjust offsets + { + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Settings indexSettings = Settings.builder() + .put( + IndexMetadata.SETTING_VERSION_CREATED, + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0) + ) + ) + .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph") + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); + + try ( + IndexAnalyzers indexAnalyzers = new AnalysisModule( + TestEnvironment.newEnvironment(settings), + Collections.singletonList(new CommonAnalysisPlugin()), + new StablePluginsRegistry() + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings) + ) { + + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 0 }, new int[] { 4, 4 }); + + } + } + + // Afger 7.3 we do adjust offsets + { + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph") + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); - try ( - IndexAnalyzers indexAnalyzers = new AnalysisModule( - TestEnvironment.newEnvironment(settings), - Collections.singletonList(new CommonAnalysisPlugin()), - new StablePluginsRegistry() - ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings) - ) { + try ( + IndexAnalyzers indexAnalyzers = new AnalysisModule( + TestEnvironment.newEnvironment(settings), + Collections.singletonList(new CommonAnalysisPlugin()), + new StablePluginsRegistry() + ).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings) + ) { - NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); - assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 1 }, new int[] { 1, 4 }); + NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 1 }, new int[] { 1, 4 }); + } } } } From 5c4348b9fc54278bc80f8375675c538020142d0c Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Wed, 22 Jan 2025 13:17:42 +0200 Subject: [PATCH 4/5] Revert code --- .../common/EdgeNGramTokenizerTests.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java index 27ff2548c126e..1b640f85cb674 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java @@ -61,7 +61,7 @@ public void testPreConfiguredTokenizer() throws IOException { try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "test", new String[]{"t"}); + assertAnalyzesTo(analyzer, "test", new String[] { "t" }); } } @@ -75,7 +75,7 @@ public void testPreConfiguredTokenizer() throws IOException { try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "test", new String[]{"t"}); + assertAnalyzesTo(analyzer, "test", new String[] { "t" }); } } @@ -84,26 +84,21 @@ public void testPreConfiguredTokenizer() throws IOException { try (IndexAnalyzers indexAnalyzers = buildAnalyzers(IndexVersion.current(), "edge_ngram")) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"}); + assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" }); } } // Check deprecated name as well, needs version before 8.0 because throws IAE after that { - try ( - IndexAnalyzers indexAnalyzers = buildAnalyzers( - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_3_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) - ), - "edgeNGram" - ) - ) { + IndexVersion version = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_7_3_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) + ); + try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) { NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer"); assertNotNull(analyzer); - assertAnalyzesTo(analyzer, "test", new String[]{"t", "te"}); - + assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" }); } } } From 8da82408ee6bb049c8c115b7c9b3f3ddea226100 Mon Sep 17 00:00:00 2001 From: Dimitris Rempapis Date: Wed, 22 Jan 2025 15:55:06 +0200 Subject: [PATCH 5/5] updagte code after review --- .../analysis/common/CommonAnalysisPluginTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java index 9972d58b2dcc1..0c88cd0c53747 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java @@ -58,7 +58,7 @@ public void testNGramFilterInCustomAnalyzerDeprecationError() throws IOException .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put( IndexMetadata.SETTING_VERSION_CREATED, - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0) + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_6_0) ) .put("index.analysis.analyzer.custom_analyzer.type", "custom") .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard") @@ -107,7 +107,7 @@ public void testEdgeNGramFilterInCustomAnalyzerDeprecationError() throws IOExcep .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) .put( IndexMetadata.SETTING_VERSION_CREATED, - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0) + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_6_0) ) .put("index.analysis.analyzer.custom_analyzer.type", "custom") .put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard") @@ -133,13 +133,13 @@ public void testNGramTokenizerDeprecation() throws IOException { doTestPrebuiltTokenizerDeprecation( "nGram", "ngram", - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2), + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2), false ); doTestPrebuiltTokenizerDeprecation( "edgeNGram", "edge_ngram", - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2), + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2), false ); doTestPrebuiltTokenizerDeprecation( @@ -185,13 +185,13 @@ public void testNGramTokenizerDeprecation() throws IOException { doTestCustomTokenizerDeprecation( "nGram", "ngram", - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2), + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2), false ); doTestCustomTokenizerDeprecation( "edgeNGram", "edge_ngram", - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2), + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2), false ); doTestCustomTokenizerDeprecation(