Skip to content

Commit b1b4282

Browse files
author
Christoph Büscher
committed
Make Multiplexer inherit filter chains analysis mode (#50662)
Currently, if an updateable synonym filter is included in a multiplexer filter, it is not reloaded via the _reload_search_analyzers because the multiplexer itself doesn't pass on the analysis mode of the filters it contains, so its not recognized as "updateable" in itself. Instead we can check and merge the AnalysisMode settings of all filters in the multiplexer and use the resulting mode (e.g. search-time only) for the multiplexer itself, thus making any synonym filters contained in it reloadable. This, of course, will also make the analyzers using the multiplexer be usable at search-time only. Closes #50554
1 parent 78c9eee commit b1b4282

File tree

3 files changed

+135
-42
lines changed

3 files changed

+135
-42
lines changed

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MultiplexerTokenFilterFactory.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.env.Environment;
3333
import org.elasticsearch.index.IndexSettings;
3434
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
35+
import org.elasticsearch.index.analysis.AnalysisMode;
3536
import org.elasticsearch.index.analysis.CharFilterFactory;
3637
import org.elasticsearch.index.analysis.TokenFilterFactory;
3738
import org.elasticsearch.index.analysis.TokenizerFactory;
@@ -84,12 +85,15 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory token
8485
if (preserveOriginal) {
8586
filters.add(IDENTITY_FILTER);
8687
}
88+
// also merge and transfer token filter analysis modes with analyzer
89+
AnalysisMode mode = AnalysisMode.ALL;
8790
for (String filter : filterNames) {
8891
String[] parts = Strings.tokenizeToStringArray(filter, ",");
8992
if (parts.length == 1) {
9093
TokenFilterFactory factory = resolveFilterFactory(allFilters, parts[0]);
9194
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, previousTokenFilters, allFilters);
9295
filters.add(factory);
96+
mode = mode.merge(factory.getAnalysisMode());
9397
} else {
9498
List<TokenFilterFactory> existingChain = new ArrayList<>(previousTokenFilters);
9599
List<TokenFilterFactory> chain = new ArrayList<>();
@@ -98,10 +102,12 @@ public TokenFilterFactory getChainAwareTokenFilterFactory(TokenizerFactory token
98102
factory = factory.getChainAwareTokenFilterFactory(tokenizer, charFilters, existingChain, allFilters);
99103
chain.add(factory);
100104
existingChain.add(factory);
105+
mode = mode.merge(factory.getAnalysisMode());
101106
}
102107
filters.add(chainFilters(filter, chain));
103108
}
104109
}
110+
final AnalysisMode analysisMode = mode;
105111

106112
return new TokenFilterFactory() {
107113
@Override
@@ -133,6 +139,11 @@ public TokenFilterFactory getSynonymFilter() {
133139
+ "] cannot be used to parse synonyms unless [preserve_original] is [true]");
134140
}
135141
}
142+
143+
@Override
144+
public AnalysisMode getAnalysisMode() {
145+
return analysisMode;
146+
}
136147
};
137148
}
138149

‎server/src/main/java/org/elasticsearch/index/analysis/AnalysisMode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,5 @@ public String getReadableName() {
7878
* <li>INDEX_TIME.merge(SEARCH_TIME) throws an {@link IllegalStateException}</li>
7979
* </ul>
8080
*/
81-
abstract AnalysisMode merge(AnalysisMode other);
81+
public abstract AnalysisMode merge(AnalysisMode other);
8282
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/ReloadSynonymAnalyzerTests.java‎

Lines changed: 123 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -43,35 +43,25 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
4343

4444
public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
4545
String synonymsFileName = "synonyms.txt";
46-
Path configDir = node().getEnvironment().configFile();
47-
if (Files.exists(configDir) == false) {
48-
Files.createDirectory(configDir);
49-
}
50-
Path synonymsFile = configDir.resolve(synonymsFileName);
51-
if (Files.exists(synonymsFile) == false) {
52-
Files.createFile(synonymsFile);
53-
}
54-
try (PrintWriter out = new PrintWriter(
55-
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
56-
out.println("foo, baz");
57-
}
46+
Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz");
5847

5948
final String indexName = "test";
6049
final String synonymAnalyzerName = "synonym_analyzer";
6150
final String synonymGraphAnalyzerName = "synonym_graph_analyzer";
62-
assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
63-
.put("index.number_of_shards", 5)
64-
.put("index.number_of_replicas", 0)
65-
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard")
66-
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter")
67-
.put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard")
68-
.putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter")
69-
.put("analysis.filter.synonym_filter.type", "synonym")
70-
.put("analysis.filter.synonym_filter.updateable", "true")
71-
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
72-
.put("analysis.filter.synonym_graph_filter.type", "synonym_graph")
73-
.put("analysis.filter.synonym_graph_filter.updateable", "true")
74-
.put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName))
51+
assertAcked(client().admin().indices().prepareCreate(indexName)
52+
.setSettings(Settings.builder()
53+
.put("index.number_of_shards", 5)
54+
.put("index.number_of_replicas", 0)
55+
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "standard")
56+
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "lowercase", "synonym_filter")
57+
.put("analysis.analyzer." + synonymGraphAnalyzerName + ".tokenizer", "standard")
58+
.putList("analysis.analyzer." + synonymGraphAnalyzerName + ".filter", "lowercase", "synonym_graph_filter")
59+
.put("analysis.filter.synonym_filter.type", "synonym")
60+
.put("analysis.filter.synonym_filter.updateable", "true")
61+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
62+
.put("analysis.filter.synonym_graph_filter.type", "synonym_graph")
63+
.put("analysis.filter.synonym_graph_filter.updateable", "true")
64+
.put("analysis.filter.synonym_graph_filter.synonyms_path", synonymsFileName))
7565
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName));
7666

7767
client().prepareIndex(indexName, "_doc", "1").setSource("field", "Foo").get();
@@ -84,8 +74,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
8474

8575
{
8676
for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) {
87-
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName)
88-
.get();
77+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get();
8978
assertEquals(2, analyzeResponse.getTokens().size());
9079
Set<String> tokens = new HashSet<>();
9180
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
@@ -109,8 +98,7 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
10998

11099
{
111100
for (String analyzerName : new String[] { synonymAnalyzerName, synonymGraphAnalyzerName }) {
112-
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName)
113-
.get();
101+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(analyzerName).get();
114102
assertEquals(3, analyzeResponse.getTokens().size());
115103
Set<String> tokens = new HashSet<>();
116104
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
@@ -126,8 +114,69 @@ public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
126114
assertHitCount(response, 1L);
127115
}
128116

117+
public void testSynonymsInMultiplexerUpdateable() throws FileNotFoundException, IOException {
118+
String synonymsFileName = "synonyms.txt";
119+
Path synonymsFile = setupSynonymsFile(synonymsFileName, "foo, baz");
120+
121+
final String indexName = "test";
122+
final String synonymAnalyzerName = "synonym_in_multiplexer_analyzer";
123+
assertAcked(client().admin().indices().prepareCreate(indexName)
124+
.setSettings(Settings.builder()
125+
.put("index.number_of_shards", 5)
126+
.put("index.number_of_replicas", 0)
127+
.put("analysis.analyzer." + synonymAnalyzerName + ".tokenizer", "whitespace")
128+
.putList("analysis.analyzer." + synonymAnalyzerName + ".filter", "my_multiplexer")
129+
.put("analysis.filter.synonym_filter.type", "synonym")
130+
.put("analysis.filter.synonym_filter.updateable", "true")
131+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
132+
.put("analysis.filter.my_multiplexer.type", "multiplexer")
133+
.putList("analysis.filter.my_multiplexer.filters", "synonym_filter"))
134+
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=" + synonymAnalyzerName));
135+
136+
client().prepareIndex(indexName, "_doc", "1").setSource("field", "foo").get();
137+
assertNoFailures(client().admin().indices().prepareRefresh(indexName).execute().actionGet());
138+
139+
SearchResponse response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
140+
assertHitCount(response, 1L);
141+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
142+
assertHitCount(response, 0L);
143+
144+
Response analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get();
145+
assertEquals(2, analyzeResponse.getTokens().size());
146+
final Set<String> tokens = new HashSet<>();
147+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
148+
assertTrue(tokens.contains("foo"));
149+
assertTrue(tokens.contains("baz"));
150+
151+
// now update synonyms file and trigger reloading
152+
try (PrintWriter out = new PrintWriter(
153+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
154+
out.println("foo, baz, buzz");
155+
}
156+
ReloadAnalyzersResponse reloadResponse = client().execute(ReloadAnalyzerAction.INSTANCE, new ReloadAnalyzersRequest(indexName))
157+
.actionGet();
158+
assertNoFailures(reloadResponse);
159+
Set<String> reloadedAnalyzers = reloadResponse.getReloadDetails().get(indexName).getReloadedAnalyzers();
160+
assertEquals(1, reloadedAnalyzers.size());
161+
assertTrue(reloadedAnalyzers.contains(synonymAnalyzerName));
162+
163+
analyzeResponse = client().admin().indices().prepareAnalyze(indexName, "foo").setAnalyzer(synonymAnalyzerName).get();
164+
assertEquals(3, analyzeResponse.getTokens().size());
165+
tokens.clear();
166+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
167+
assertTrue(tokens.contains("foo"));
168+
assertTrue(tokens.contains("baz"));
169+
assertTrue(tokens.contains("buzz"));
170+
171+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "baz")).get();
172+
assertHitCount(response, 1L);
173+
response = client().prepareSearch(indexName).setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
174+
assertHitCount(response, 1L);
175+
}
176+
129177
public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundException, IOException {
130178
String synonymsFileName = "synonyms.txt";
179+
setupSynonymsFile(synonymsFileName, "foo, baz");
131180
Path configDir = node().getEnvironment().configFile();
132181
if (Files.exists(configDir) == false) {
133182
Files.createDirectory(configDir);
@@ -143,20 +192,53 @@ public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundExcep
143192

144193
final String indexName = "test";
145194
final String analyzerName = "my_synonym_analyzer";
195+
146196
MapperException ex = expectThrows(MapperException.class, () -> client().admin().indices().prepareCreate(indexName)
147197
.setSettings(Settings.builder()
148-
.put("index.number_of_shards", 5)
149-
.put("index.number_of_replicas", 0)
150-
.put("analysis.analyzer." + analyzerName + ".tokenizer", "standard")
151-
.putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter")
152-
.put("analysis.filter.synonym_filter.type", "synonym")
153-
.put("analysis.filter.synonym_filter.updateable", "true")
154-
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName))
198+
.put("index.number_of_shards", 5)
199+
.put("index.number_of_replicas", 0)
200+
.put("analysis.analyzer." + analyzerName + ".tokenizer", "standard")
201+
.putList("analysis.analyzer." + analyzerName + ".filter", "lowercase", "synonym_filter")
202+
.put("analysis.filter.synonym_filter.type", "synonym")
203+
.put("analysis.filter.synonym_filter.updateable", "true")
204+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName))
155205
.addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get());
156206

157-
assertEquals(
158-
"Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] "
159-
+ "contains filters [synonym_filter] that are not allowed to run in all mode.",
160-
ex.getMessage());
207+
assertEquals("Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] "
208+
+ "contains filters [synonym_filter] that are not allowed to run in all mode.", ex.getMessage());
209+
210+
// same for synonym filters in multiplexer chain
211+
ex = expectThrows(MapperException.class,
212+
() -> client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder()
213+
.put("index.number_of_shards", 5)
214+
.put("index.number_of_replicas", 0)
215+
.put("analysis.analyzer." + analyzerName + ".tokenizer", "whitespace")
216+
.putList("analysis.analyzer." + analyzerName + ".filter", "my_multiplexer")
217+
.put("analysis.filter.synonym_filter.type", "synonym")
218+
.put("analysis.filter.synonym_filter.updateable", "true")
219+
.put("analysis.filter.synonym_filter.synonyms_path", synonymsFileName)
220+
.put("analysis.filter.my_multiplexer.type", "multiplexer")
221+
.putList("analysis.filter.my_multiplexer.filters", "synonym_filter"))
222+
.addMapping("_doc", "field", "type=text,analyzer=" + analyzerName).get());
223+
224+
assertEquals("Failed to parse mapping [_doc]: analyzer [my_synonym_analyzer] "
225+
+ "contains filters [my_multiplexer] that are not allowed to run in all mode.", ex.getMessage());
161226
}
162-
}
227+
228+
private Path setupSynonymsFile(String synonymsFileName, String content) throws IOException {
229+
Path configDir = node().getEnvironment().configFile();
230+
if (Files.exists(configDir) == false) {
231+
Files.createDirectory(configDir);
232+
}
233+
Path synonymsFile = configDir.resolve(synonymsFileName);
234+
if (Files.exists(synonymsFile) == false) {
235+
Files.createFile(synonymsFile);
236+
}
237+
try (PrintWriter out = new PrintWriter(
238+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
239+
out.println(content);
240+
}
241+
return synonymsFile;
242+
}
243+
244+
}

0 commit comments

Comments
 (0)