Skip to content

Commit 96e915e

Browse files
author
Christoph Büscher
committed
Make limit on number of expanded fields configurable
Currently we introduced a hard limit of 1024 to the number of fields a query can be expanded to in elastic#26541. Instead of using a hard limit, we should make this configurable. This change removes the hard limit check and uses the existing `max_clause_count` setting instead. Closes elastic#34778
1 parent 85f8458 commit 96e915e

File tree

5 files changed

+59
-16
lines changed

5 files changed

+59
-16
lines changed

‎docs/reference/query-dsl/query-string-query.asciidoc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,9 @@ The `query_string` top level parameters include:
6060
specified. Defaults to the `index.query.default_field` index settings, which in
6161
turn defaults to `*`. `*` extracts all fields in the mapping that are eligible
6262
to term queries and filters the metadata fields. All extracted fields are then
63-
combined to build a query when no prefix field is provided. There is a limit of
64-
no more than 1024 fields being queried at once.
63+
combined to build a query when no prefix field is provided. There is a limit on
64+
the number of fields that can be queried at once which is defined by the
65+
`indices.query.bool.max_clause_count` <<search-settings>> which defaults to 1024.
6566

6667
|`default_operator` |The default operator used if no explicit operator
6768
is specified. For example, with a default operator of `OR`, the query

‎docs/reference/query-dsl/simple-query-string-query.asciidoc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ The `simple_query_string` top level parameters include:
3131
|`fields` |The fields to perform the parsed query against. Defaults to the
3232
`index.query.default_field` index settings, which in turn defaults to `*`. `*`
3333
extracts all fields in the mapping that are eligible to term queries and filters
34-
the metadata fields. There is a limit of no more than 1024 fields being queried
35-
at once.
34+
the metadata fields. There is a limit on the number of fields that can be queried
35+
at once which is defined by the `indices.query.bool.max_clause_count` <<search-settings>>
36+
which defaults to 1024.
3637

3738
|`default_operator` |The default operator used if no explicit operator
3839
is specified. For example, with a default operator of `OR`, the query

‎server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.index.mapper.MappedFieldType;
2525
import org.elasticsearch.index.query.QueryShardContext;
2626
import org.elasticsearch.index.query.QueryShardException;
27+
import org.elasticsearch.search.SearchModule;
2728

2829
import java.util.Collection;
2930
import java.util.HashMap;
@@ -85,7 +86,7 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
8586
!multiField, !allField, fieldSuffix);
8687
resolvedFields.putAll(fieldMap);
8788
}
88-
checkForTooManyFields(resolvedFields);
89+
checkForTooManyFields(resolvedFields, context);
8990
return resolvedFields;
9091
}
9192

@@ -141,7 +142,7 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
141142
if (acceptAllTypes == false) {
142143
try {
143144
fieldType.termQuery("", context);
144-
} catch (QueryShardException |UnsupportedOperationException e) {
145+
} catch (QueryShardException | UnsupportedOperationException e) {
145146
// field type is never searchable with term queries (eg. geo point): ignore
146147
continue;
147148
} catch (IllegalArgumentException |ElasticsearchParseException e) {
@@ -150,13 +151,14 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
150151
}
151152
fields.put(fieldName, weight);
152153
}
153-
checkForTooManyFields(fields);
154+
checkForTooManyFields(fields, context);
154155
return fields;
155156
}
156157

157-
private static void checkForTooManyFields(Map<String, Float> fields) {
158-
if (fields.size() > 1024) {
159-
throw new IllegalArgumentException("field expansion matches too many fields, limit: 1024, got: " + fields.size());
158+
private static void checkForTooManyFields(Map<String, Float> fields, QueryShardContext context) {
159+
Integer limit = SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.get(context.getIndexSettings().getSettings());
160+
if (fields.size() > limit) {
161+
throw new IllegalArgumentException("field expansion matches too many fields, limit: " + limit + ", got: " + fields.size());
160162
}
161163
}
162164
}

‎server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@
3030
import org.elasticsearch.index.query.QueryStringQueryBuilder;
3131
import org.elasticsearch.search.SearchHit;
3232
import org.elasticsearch.search.SearchHits;
33+
import org.elasticsearch.search.SearchModule;
3334
import org.elasticsearch.test.ESIntegTestCase;
3435
import org.junit.Before;
36+
import org.junit.BeforeClass;
3537

3638
import java.io.IOException;
3739
import java.util.ArrayList;
@@ -51,13 +53,28 @@
5153

5254
public class QueryStringIT extends ESIntegTestCase {
5355

56+
private static int CLUSTER_MAX_CLAUSE_COUNT;
57+
58+
@BeforeClass
59+
public static void createRandomClusterSetting() {
60+
CLUSTER_MAX_CLAUSE_COUNT = randomIntBetween(500, 1500);
61+
}
62+
5463
@Before
5564
public void setup() throws Exception {
5665
String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
5766
prepareCreate("test").setSource(indexBody, XContentType.JSON).get();
5867
ensureGreen("test");
5968
}
6069

70+
@Override
71+
protected Settings nodeSettings(int nodeOrdinal) {
72+
return Settings.builder()
73+
.put(super.nodeSettings(nodeOrdinal))
74+
.put(SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
75+
.build();
76+
}
77+
6178
public void testBasicAllQuery() throws Exception {
6279
List<IndexRequestBuilder> reqs = new ArrayList<>();
6380
reqs.add(client().prepareIndex("test", "_doc", "1").setSource("f1", "foo bar baz"));
@@ -253,15 +270,16 @@ public void testLimitOnExpandedFields() throws Exception {
253270
builder.startObject();
254271
builder.startObject("type1");
255272
builder.startObject("properties");
256-
for (int i = 0; i < 1025; i++) {
273+
for (int i = 0; i < CLUSTER_MAX_CLAUSE_COUNT + 1; i++) {
257274
builder.startObject("field" + i).field("type", "text").endObject();
258275
}
259276
builder.endObject(); // properties
260277
builder.endObject(); // type1
261278
builder.endObject();
262279

263280
assertAcked(prepareCreate("toomanyfields")
264-
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200))
281+
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
282+
CLUSTER_MAX_CLAUSE_COUNT + 100))
265283
.addMapping("type1", builder));
266284

267285
client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get();
@@ -276,7 +294,8 @@ public void testLimitOnExpandedFields() throws Exception {
276294
client().prepareSearch("toomanyfields").setQuery(qb).get();
277295
});
278296
assertThat(ExceptionsHelper.detailedMessage(e),
279-
containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
297+
containsString("field expansion matches too many fields, limit: " + CLUSTER_MAX_CLAUSE_COUNT + ", got: "
298+
+ (CLUSTER_MAX_CLAUSE_COUNT + 1)));
280299
}
281300

282301
public void testFieldAlias() throws Exception {

‎server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@
4242
import org.elasticsearch.plugins.Plugin;
4343
import org.elasticsearch.search.SearchHit;
4444
import org.elasticsearch.search.SearchHits;
45+
import org.elasticsearch.search.SearchModule;
4546
import org.elasticsearch.search.builder.SearchSourceBuilder;
4647
import org.elasticsearch.test.ESIntegTestCase;
48+
import org.junit.BeforeClass;
4749

4850
import java.io.IOException;
4951
import java.util.ArrayList;
@@ -76,6 +78,22 @@
7678
* Tests for the {@code simple_query_string} query
7779
*/
7880
public class SimpleQueryStringIT extends ESIntegTestCase {
81+
82+
private static int CLUSTER_MAX_CLAUSE_COUNT;
83+
84+
@BeforeClass
85+
public static void createRandomClusterSetting() {
86+
CLUSTER_MAX_CLAUSE_COUNT = randomIntBetween(500, 1500);
87+
}
88+
89+
@Override
90+
protected Settings nodeSettings(int nodeOrdinal) {
91+
return Settings.builder()
92+
.put(super.nodeSettings(nodeOrdinal))
93+
.put(SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING.getKey(), CLUSTER_MAX_CLAUSE_COUNT)
94+
.build();
95+
}
96+
7997
@Override
8098
protected Collection<Class<? extends Plugin>> nodePlugins() {
8199
return Collections.singletonList(MockAnalysisPlugin.class);
@@ -559,15 +577,16 @@ public void testLimitOnExpandedFields() throws Exception {
559577
builder.startObject();
560578
builder.startObject("type1");
561579
builder.startObject("properties");
562-
for (int i = 0; i < 1025; i++) {
580+
for (int i = 0; i < CLUSTER_MAX_CLAUSE_COUNT + 1; i++) {
563581
builder.startObject("field" + i).field("type", "text").endObject();
564582
}
565583
builder.endObject(); // properties
566584
builder.endObject(); // type1
567585
builder.endObject();
568586

569587
assertAcked(prepareCreate("toomanyfields")
570-
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200))
588+
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
589+
CLUSTER_MAX_CLAUSE_COUNT + 100))
571590
.addMapping("type1", builder));
572591

573592
client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get();
@@ -582,7 +601,8 @@ public void testLimitOnExpandedFields() throws Exception {
582601
client().prepareSearch("toomanyfields").setQuery(qb).get();
583602
});
584603
assertThat(ExceptionsHelper.detailedMessage(e),
585-
containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
604+
containsString("field expansion matches too many fields, limit: " + CLUSTER_MAX_CLAUSE_COUNT + ", got: "
605+
+ (CLUSTER_MAX_CLAUSE_COUNT + 1)));
586606
}
587607

588608
public void testFieldAlias() throws Exception {

0 commit comments

Comments
 (0)