Skip to content

Reintroduce common-analysis tests #120604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 31 commits into from
Jan 22, 2025
Merged

Reintroduce common-analysis tests #120604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
9d9b6af
unmute tests
drempapis Jan 3, 2025
02ddf82
revert
drempapis Jan 3, 2025
bf79ef3
Merge remote-tracking branch 'upstream/main'
drempapis Jan 3, 2025
f1c91bd
Merge remote-tracking branch 'upstream/main'
drempapis Jan 7, 2025
ba67bff
Merge remote-tracking branch 'upstream/main'
drempapis Jan 7, 2025
2c3654a
Merge remote-tracking branch 'upstream/main'
drempapis Jan 8, 2025
58d4762
Merge remote-tracking branch 'upstream/main'
drempapis Jan 8, 2025
bc38090
Merge remote-tracking branch 'upstream/main'
drempapis Jan 9, 2025
ef0447b
Merge remote-tracking branch 'upstream/main'
drempapis Jan 9, 2025
fe009d7
Merge remote-tracking branch 'upstream/main'
drempapis Jan 10, 2025
a747a40
Merge remote-tracking branch 'upstream/main'
drempapis Jan 10, 2025
f3e47ae
Merge remote-tracking branch 'upstream/main'
drempapis Jan 10, 2025
2bc0107
Merge remote-tracking branch 'upstream/main'
drempapis Jan 13, 2025
f3b3d00
Merge remote-tracking branch 'upstream/main'
drempapis Jan 13, 2025
f52789e
Merge remote-tracking branch 'upstream/main'
drempapis Jan 14, 2025
ec243b7
Merge remote-tracking branch 'upstream/main'
drempapis Jan 14, 2025
f93eb9b
Merge remote-tracking branch 'upstream/main'
drempapis Jan 16, 2025
2777916
Merge remote-tracking branch 'upstream/main'
drempapis Jan 16, 2025
623bd7b
Merge remote-tracking branch 'upstream/main'
drempapis Jan 16, 2025
3fee6af
Merge remote-tracking branch 'upstream/main'
drempapis Jan 16, 2025
af3fff9
Merge remote-tracking branch 'upstream/main'
drempapis Jan 17, 2025
a41bbad
Merge remote-tracking branch 'upstream/main'
drempapis Jan 17, 2025
8115a61
Merge remote-tracking branch 'upstream/main'
drempapis Jan 20, 2025
6b2361e
Merge remote-tracking branch 'upstream/main'
drempapis Jan 20, 2025
41dcc1c
Merge remote-tracking branch 'upstream/main'
drempapis Jan 21, 2025
2df7f62
Merge remote-tracking branch 'upstream/main'
drempapis Jan 21, 2025
61bed25
Revert tests
drempapis Jan 21, 2025
5c4348b
Revert code
drempapis Jan 22, 2025
b5b71fa
Merge branch 'main' into test/Analysis_tests_stack_v9
drempapis Jan 22, 2025
8da8240
updagte code after review
drempapis Jan 22, 2025
b2fa973
Merge branch 'main' into test/Analysis_tests_stack_v9
drempapis Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions ...sis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisPluginTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public void testNGramFilterInCustomAnalyzerDeprecationError() throws IOException
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(
IndexMetadata.SETTING_VERSION_CREATED,
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0)
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_6_0)
)
.put("index.analysis.analyzer.custom_analyzer.type", "custom")
.put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
Expand Down Expand Up @@ -107,7 +107,7 @@ public void testEdgeNGramFilterInCustomAnalyzerDeprecationError() throws IOExcep
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(
IndexMetadata.SETTING_VERSION_CREATED,
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_6_0)
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_6_0)
)
.put("index.analysis.analyzer.custom_analyzer.type", "custom")
.put("index.analysis.analyzer.custom_analyzer.tokenizer", "standard")
Expand All @@ -133,13 +133,13 @@ public void testNGramTokenizerDeprecation() throws IOException {
doTestPrebuiltTokenizerDeprecation(
"nGram",
"ngram",
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2),
false
);
doTestPrebuiltTokenizerDeprecation(
"edgeNGram",
"edge_ngram",
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2),
false
);
doTestPrebuiltTokenizerDeprecation(
Expand Down Expand Up @@ -185,13 +185,13 @@ public void testNGramTokenizerDeprecation() throws IOException {
doTestCustomTokenizerDeprecation(
"nGram",
"ngram",
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2),
false
);
doTestCustomTokenizerDeprecation(
"edgeNGram",
"edge_ngram",
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_0_0, IndexVersions.V_7_5_2),
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.V_7_5_2),
false
);
doTestCustomTokenizerDeprecation(
Expand Down
54 changes: 50 additions & 4 deletions ...lysis-common/src/test/java/org/elasticsearch/analysis/common/EdgeNGramTokenizerTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@
import org.elasticsearch.index.IndexService.IndexCreationContext;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.indices.analysis.AnalysisModule;
import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
import org.elasticsearch.test.ESTokenStreamTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.index.IndexVersionUtils;

import java.io.IOException;
import java.io.StringReader;
Expand All @@ -49,12 +51,56 @@ private IndexAnalyzers buildAnalyzers(IndexVersion version, String tokenizer) th
}

public void testPreConfiguredTokenizer() throws IOException {
try (IndexAnalyzers indexAnalyzers = buildAnalyzers(IndexVersion.current(), "edge_ngram")) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" });
// Before 7.3 we return ngrams of length 1 only
{
IndexVersion version = IndexVersionUtils.randomVersionBetween(
random(),
IndexVersions.MINIMUM_READONLY_COMPATIBLE,
Copy link
Contributor Author

@drempapis drempapis Jan 22, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have replaced the IndexVersions.V_7_0_0 with IndexVersions.MINIMUM_READONLY_COMPATIBLE. If you think that it is ok, we should also apply it to CommonAnalysisPluginTests and SynonymsAnalysisTests

Copy link
Member

@cbuescher cbuescher Jan 22, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think that is okay. I see the tests from #112570 regarding CommonAnalysisPluginTests and SynonymAnalysisTests are already rolled back, would you mind doing the suggested changes in this PR just for completeness?

Copy link
Contributor Author

@drempapis drempapis Jan 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you, @cbuescher, for the review. Yes, I'll make the update in this PR.

IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0)
);
try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "test", new String[] { "t" });
}
}

// Check deprecated name as well
{
IndexVersion version = IndexVersionUtils.randomVersionBetween(
random(),
IndexVersions.MINIMUM_READONLY_COMPATIBLE,
IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0)
);
try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edgeNGram")) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "test", new String[] { "t" });
}
}

// Afterwards, we return ngrams of length 1 and 2, to match the default factory settings
{
try (IndexAnalyzers indexAnalyzers = buildAnalyzers(IndexVersion.current(), "edge_ngram")) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" });
}
}

// Check deprecated name as well, needs version before 8.0 because throws IAE after that
{
IndexVersion version = IndexVersionUtils.randomVersionBetween(
random(),
IndexVersions.V_7_3_0,
IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0)
);
try (IndexAnalyzers indexAnalyzers = buildAnalyzers(version, "edge_ngram")) {
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "test", new String[] { "t", "te" });
}
}
}

public void testCustomTokenChars() throws IOException {
Expand Down
71 changes: 54 additions & 17 deletions ...est/java/org/elasticsearch/analysis/common/WordDelimiterGraphTokenFilterFactoryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.elasticsearch.index.IndexService.IndexCreationContext;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.analysis.AnalysisTestsHelper;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
Expand All @@ -25,6 +26,7 @@
import org.elasticsearch.plugins.scanners.StablePluginsRegistry;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.index.IndexVersionUtils;

import java.io.IOException;
import java.io.StringReader;
Expand Down Expand Up @@ -182,26 +184,61 @@ public void testIgnoreKeywords() throws IOException {
}

public void testPreconfiguredFilter() throws IOException {
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
// Before 7.3 we don't adjust offsets
{
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(
IndexMetadata.SETTING_VERSION_CREATED,
IndexVersionUtils.randomVersionBetween(
random(),
IndexVersions.MINIMUM_READONLY_COMPATIBLE,
IndexVersionUtils.getPreviousVersion(IndexVersions.V_7_3_0)
)
)
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);

try (
IndexAnalyzers indexAnalyzers = new AnalysisModule(
TestEnvironment.newEnvironment(settings),
Collections.singletonList(new CommonAnalysisPlugin()),
new StablePluginsRegistry()
).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings)
) {

NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 0 }, new int[] { 4, 4 });

}
}

// Afger 7.3 we do adjust offsets
{
Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
Settings indexSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
.put("index.analysis.analyzer.my_analyzer.tokenizer", "standard")
.putList("index.analysis.analyzer.my_analyzer.filter", "word_delimiter_graph")
.build();
IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);

try (
IndexAnalyzers indexAnalyzers = new AnalysisModule(
TestEnvironment.newEnvironment(settings),
Collections.singletonList(new CommonAnalysisPlugin()),
new StablePluginsRegistry()
).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings)
) {
try (
IndexAnalyzers indexAnalyzers = new AnalysisModule(
TestEnvironment.newEnvironment(settings),
Collections.singletonList(new CommonAnalysisPlugin()),
new StablePluginsRegistry()
).getAnalysisRegistry().build(IndexCreationContext.CREATE_INDEX, idxSettings)
) {

NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 1 }, new int[] { 1, 4 });
NamedAnalyzer analyzer = indexAnalyzers.get("my_analyzer");
assertNotNull(analyzer);
assertAnalyzesTo(analyzer, "h100", new String[] { "h", "100" }, new int[] { 0, 1 }, new int[] { 1, 4 });

}
}
}
}