Skip to content

Commit 2032881

Browse files
jasontedorjasontedor
committed
Merge branch 'master' into ccr
* master: (22 commits) Update Tika version to 1.15 Aggregations: bucket_sort pipeline aggregation (#27152) Introduce templating support to timezone/locale in DateProcessor (#27089) Increase logging on qa:mixed-cluster tests Update to AWS SDK 1.11.223 (#27278) Improve error message for parse failures of completion fields (#27297) Ensure external refreshes will also refresh internal searcher to minimize segment creation (#27253) Remove optimisations to reuse objects when applying a new `ClusterState` (#27317) Decouple `ChannelFactory` from Tcp classes (#27286) Fix find remote when building BWC Remove colons from task and configuration names Add unreleased 5.6.5 version number testCreateSplitIndexToN: do not set `routing_partition_size` to >= `number_of_routing_shards` Snapshot/Restore: better handle incorrect chunk_size settings in FS repo (#26844) Add limits for ngram and shingle settings (#27211) (#27318) Correct comment in index shard test Roll translog generation on primary promotion ObjectParser: Replace IllegalStateException with ParsingException (#27302) scripted_metric _agg parameter disappears if params are provided (#27159) Update discovery-ec2.asciidoc ...
2 parents ddd3ed4 + ac5fd6a commit 2032881

File tree

106 files changed

+2225
-293
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

106 files changed

+2225
-293
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ task verifyVersions {
186186
* after the backport of the backcompat code is complete.
187187
*/
188188
allprojects {
189-
ext.bwc_tests_enabled = false
189+
ext.bwc_tests_enabled = true
190190
}
191191

192192
task verifyBwcTestsEnabled {

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ class BuildPlugin implements Plugin<Project> {
239239

240240
/** Return the configuration name used for finding transitive deps of the given dependency. */
241241
private static String transitiveDepConfigName(String groupId, String artifactId, String version) {
242-
return "_transitive_${groupId}:${artifactId}:${version}"
242+
return "_transitive_${groupId}_${artifactId}_${version}"
243243
}
244244

245245
/**

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,7 @@ class ClusterFormationTasks {
425425

426426
Project pluginProject = plugin.getValue()
427427
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
428-
String configurationName = "_plugin_${prefix}_${pluginProject.path}"
428+
String configurationName = pluginConfigurationName(prefix, pluginProject)
429429
Configuration configuration = project.configurations.findByName(configurationName)
430430
if (configuration == null) {
431431
configuration = project.configurations.create(configurationName)
@@ -454,13 +454,21 @@ class ClusterFormationTasks {
454454
return copyPlugins
455455
}
456456

457+
private static String pluginConfigurationName(final String prefix, final Project project) {
458+
return "_plugin_${prefix}_${project.path}".replace(':', '_')
459+
}
460+
461+
private static String pluginBwcConfigurationName(final String prefix, final Project project) {
462+
return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
463+
}
464+
457465
/** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
458466
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
459467
Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
460468
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
461469
Project pluginProject = plugin.getValue()
462470
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
463-
String configurationName = "_plugin_bwc_${prefix}_${pluginProject.path}"
471+
String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
464472
Configuration configuration = project.configurations.findByName(configurationName)
465473
if (configuration == null) {
466474
configuration = project.configurations.create(configurationName)
@@ -499,9 +507,9 @@ class ClusterFormationTasks {
499507
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
500508
final FileCollection pluginZip;
501509
if (node.nodeVersion != VersionProperties.elasticsearch) {
502-
pluginZip = project.configurations.getByName("_plugin_bwc_${prefix}_${plugin.path}")
510+
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
503511
} else {
504-
pluginZip = project.configurations.getByName("_plugin_${prefix}_${plugin.path}")
512+
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
505513
}
506514
// delay reading the file location until execution time by wrapping in a closure within a GString
507515
final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"

‎core/src/main/java/org/elasticsearch/Version.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ public class Version implements Comparable<Version> {
100100
public static final Version V_5_6_3 = new Version(V_5_6_3_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
101101
public static final int V_5_6_4_ID = 5060499;
102102
public static final Version V_5_6_4 = new Version(V_5_6_4_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
103+
public static final int V_5_6_5_ID = 5060599;
104+
public static final Version V_5_6_5 = new Version(V_5_6_5_ID, org.apache.lucene.util.Version.LUCENE_6_6_1);
103105
public static final int V_6_0_0_alpha1_ID = 6000001;
104106
public static final Version V_6_0_0_alpha1 =
105107
new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0);
@@ -160,6 +162,8 @@ public static Version fromId(int id) {
160162
return V_6_0_0_alpha2;
161163
case V_6_0_0_alpha1_ID:
162164
return V_6_0_0_alpha1;
165+
case V_5_6_5_ID:
166+
return V_5_6_5;
163167
case V_5_6_4_ID:
164168
return V_5_6_4;
165169
case V_5_6_3_ID:

‎core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public class ResizeAction extends Action<ResizeRequest, ResizeResponse, ResizeRe
2727

2828
public static final ResizeAction INSTANCE = new ResizeAction();
2929
public static final String NAME = "indices:admin/resize";
30-
public static final Version COMPATIBILITY_VERSION = Version.V_7_0_0_alpha1; // TODO remove this once it's backported
30+
public static final Version COMPATIBILITY_VERSION = Version.V_6_1_0; // TODO remove this once it's backported
3131

3232
private ResizeAction() {
3333
super(NAME);

‎core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
114114
IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING,
115115
IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING,
116116
IndexSettings.MAX_SCRIPT_FIELDS_SETTING,
117+
IndexSettings.MAX_NGRAM_DIFF_SETTING,
118+
IndexSettings.MAX_SHINGLE_DIFF_SETTING,
117119
IndexSettings.MAX_RESCORE_WINDOW_SETTING,
118120
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
119121
IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
@@ -150,6 +152,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
150152
EngineConfig.INDEX_CODEC_SETTING,
151153
EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,
152154
IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS,
155+
153156
// validate that built-in similarities don't get redefined
154157
Setting.groupSetting("index.similarity.", (s) -> {
155158
Map<String, Settings> groups = s.getAsGroups();

‎core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
147147
} else {
148148
token = parser.nextToken();
149149
if (token != XContentParser.Token.START_OBJECT) {
150-
throw new IllegalStateException("[" + name + "] Expected START_OBJECT but was: " + token);
150+
throw new ParsingException(parser.getTokenLocation(), "[" + name + "] Expected START_OBJECT but was: " + token);
151151
}
152152
}
153153

@@ -159,13 +159,13 @@ public Value parse(XContentParser parser, Value value, Context context) throws I
159159
fieldParser = getParser(currentFieldName);
160160
} else {
161161
if (currentFieldName == null) {
162-
throw new IllegalStateException("[" + name + "] no field found");
162+
throw new ParsingException(parser.getTokenLocation(), "[" + name + "] no field found");
163163
}
164164
if (fieldParser == null) {
165165
assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields";
166166
parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array
167167
} else {
168-
fieldParser.assertSupports(name, token, currentFieldName);
168+
fieldParser.assertSupports(name, token, currentFieldName, parser.getTokenLocation());
169169
parseSub(parser, fieldParser, currentFieldName, value, context);
170170
}
171171
fieldParser = null;
@@ -330,7 +330,7 @@ private void parseSub(XContentParser parser, FieldParser fieldParser, String cur
330330
case END_OBJECT:
331331
case END_ARRAY:
332332
case FIELD_NAME:
333-
throw new IllegalStateException("[" + name + "]" + token + " is unexpected");
333+
throw new ParsingException(parser.getTokenLocation(), "[" + name + "]" + token + " is unexpected");
334334
case VALUE_STRING:
335335
case VALUE_NUMBER:
336336
case VALUE_BOOLEAN:
@@ -361,12 +361,12 @@ private class FieldParser {
361361
this.type = type;
362362
}
363363

364-
void assertSupports(String parserName, XContentParser.Token token, String currentFieldName) {
364+
void assertSupports(String parserName, XContentParser.Token token, String currentFieldName, XContentLocation location) {
365365
if (parseField.match(currentFieldName) == false) {
366-
throw new IllegalStateException("[" + parserName + "] parsefield doesn't accept: " + currentFieldName);
366+
throw new ParsingException(location, "[" + parserName + "] parsefield doesn't accept: " + currentFieldName);
367367
}
368368
if (supportedTokens.contains(token) == false) {
369-
throw new IllegalArgumentException(
369+
throw new ParsingException(location,
370370
"[" + parserName + "] " + currentFieldName + " doesn't support values of type: " + token);
371371
}
372372
}

‎core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java‎

Lines changed: 6 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -735,7 +735,6 @@ boolean processNextCommittedClusterState(String reason) {
735735

736736
final ClusterState newClusterState = pendingStatesQueue.getNextClusterStateToProcess();
737737
final ClusterState currentState = committedState.get();
738-
final ClusterState adaptedNewClusterState;
739738
// all pending states have been processed
740739
if (newClusterState == null) {
741740
return false;
@@ -773,54 +772,23 @@ boolean processNextCommittedClusterState(String reason) {
773772
if (currentState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock())) {
774773
// its a fresh update from the master as we transition from a start of not having a master to having one
775774
logger.debug("got first state from fresh master [{}]", newClusterState.nodes().getMasterNodeId());
776-
adaptedNewClusterState = newClusterState;
777-
} else if (newClusterState.nodes().isLocalNodeElectedMaster() == false) {
778-
// some optimizations to make sure we keep old objects where possible
779-
ClusterState.Builder builder = ClusterState.builder(newClusterState);
780-
781-
// if the routing table did not change, use the original one
782-
if (newClusterState.routingTable().version() == currentState.routingTable().version()) {
783-
builder.routingTable(currentState.routingTable());
784-
}
785-
// same for metadata
786-
if (newClusterState.metaData().version() == currentState.metaData().version()) {
787-
builder.metaData(currentState.metaData());
788-
} else {
789-
// if its not the same version, only copy over new indices or ones that changed the version
790-
MetaData.Builder metaDataBuilder = MetaData.builder(newClusterState.metaData()).removeAllIndices();
791-
for (IndexMetaData indexMetaData : newClusterState.metaData()) {
792-
IndexMetaData currentIndexMetaData = currentState.metaData().index(indexMetaData.getIndex());
793-
if (currentIndexMetaData != null && currentIndexMetaData.isSameUUID(indexMetaData.getIndexUUID()) &&
794-
currentIndexMetaData.getVersion() == indexMetaData.getVersion()) {
795-
// safe to reuse
796-
metaDataBuilder.put(currentIndexMetaData, false);
797-
} else {
798-
metaDataBuilder.put(indexMetaData, false);
799-
}
800-
}
801-
builder.metaData(metaDataBuilder);
802-
}
803-
804-
adaptedNewClusterState = builder.build();
805-
} else {
806-
adaptedNewClusterState = newClusterState;
807775
}
808776

809-
if (currentState == adaptedNewClusterState) {
777+
if (currentState == newClusterState) {
810778
return false;
811779
}
812780

813-
committedState.set(adaptedNewClusterState);
781+
committedState.set(newClusterState);
814782

815783
// update failure detection only after the state has been updated to prevent race condition with handleLeaveRequest
816784
// and handleNodeFailure as those check the current state to determine whether the failure is to be handled by this node
817-
if (adaptedNewClusterState.nodes().isLocalNodeElectedMaster()) {
785+
if (newClusterState.nodes().isLocalNodeElectedMaster()) {
818786
// update the set of nodes to ping
819-
nodesFD.updateNodesAndPing(adaptedNewClusterState);
787+
nodesFD.updateNodesAndPing(newClusterState);
820788
} else {
821789
// check to see that we monitor the correct master of the cluster
822-
if (masterFD.masterNode() == null || !masterFD.masterNode().equals(adaptedNewClusterState.nodes().getMasterNode())) {
823-
masterFD.restart(adaptedNewClusterState.nodes().getMasterNode(),
790+
if (masterFD.masterNode() == null || !masterFD.masterNode().equals(newClusterState.nodes().getMasterNode())) {
791+
masterFD.restart(newClusterState.nodes().getMasterNode(),
824792
"new cluster state received and we are monitoring the wrong master [" + masterFD.masterNode() + "]");
825793
}
826794
}

‎core/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,26 @@ public final class IndexSettings {
107107
public static final Setting<Integer> MAX_SCRIPT_FIELDS_SETTING =
108108
Setting.intSetting("index.max_script_fields", 32, 0, Property.Dynamic, Property.IndexScope);
109109

110+
/**
111+
* Index setting describing for NGramTokenizer and NGramTokenFilter
112+
* the maximum difference between
113+
* max_gram (maximum length of characters in a gram) and
114+
* min_gram (minimum length of characters in a gram).
115+
* The default value is 1 as this is default difference in NGramTokenizer,
116+
* and is defensive as it prevents generating too many index terms.
117+
*/
118+
public static final Setting<Integer> MAX_NGRAM_DIFF_SETTING =
119+
Setting.intSetting("index.max_ngram_diff", 1, 0, Property.Dynamic, Property.IndexScope);
120+
121+
/**
122+
* Index setting describing for ShingleTokenFilter
123+
* the maximum difference between
124+
* max_shingle_size and min_shingle_size.
125+
* The default value is 3 is defensive as it prevents generating too many tokens.
126+
*/
127+
public static final Setting<Integer> MAX_SHINGLE_DIFF_SETTING =
128+
Setting.intSetting("index.max_shingle_diff", 3, 0, Property.Dynamic, Property.IndexScope);
129+
110130
/**
111131
* Index setting describing the maximum value of allowed `docvalue_fields`that can be retrieved
112132
* per search request. The default maximum of 100 is defensive for the reason that retrieving
@@ -239,6 +259,8 @@ public final class IndexSettings {
239259
private volatile int maxRescoreWindow;
240260
private volatile int maxDocvalueFields;
241261
private volatile int maxScriptFields;
262+
private volatile int maxNgramDiff;
263+
private volatile int maxShingleDiff;
242264
private volatile boolean TTLPurgeDisabled;
243265
/**
244266
* The maximum number of refresh listeners allows on this shard.
@@ -342,6 +364,8 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
342364
maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
343365
maxDocvalueFields = scopedSettings.get(MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
344366
maxScriptFields = scopedSettings.get(MAX_SCRIPT_FIELDS_SETTING);
367+
maxNgramDiff = scopedSettings.get(MAX_NGRAM_DIFF_SETTING);
368+
maxShingleDiff = scopedSettings.get(MAX_SHINGLE_DIFF_SETTING);
345369
TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
346370
maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
347371
maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
@@ -373,6 +397,8 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
373397
scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
374398
scopedSettings.addSettingsUpdateConsumer(MAX_DOCVALUE_FIELDS_SEARCH_SETTING, this::setMaxDocvalueFields);
375399
scopedSettings.addSettingsUpdateConsumer(MAX_SCRIPT_FIELDS_SETTING, this::setMaxScriptFields);
400+
scopedSettings.addSettingsUpdateConsumer(MAX_NGRAM_DIFF_SETTING, this::setMaxNgramDiff);
401+
scopedSettings.addSettingsUpdateConsumer(MAX_SHINGLE_DIFF_SETTING, this::setMaxShingleDiff);
376402
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
377403
scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
378404
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize);
@@ -641,6 +667,20 @@ private void setMaxDocvalueFields(int maxDocvalueFields) {
641667
this.maxDocvalueFields = maxDocvalueFields;
642668
}
643669

670+
/**
671+
* Returns the maximum allowed difference between max and min length of ngram
672+
*/
673+
public int getMaxNgramDiff() { return this.maxNgramDiff; }
674+
675+
private void setMaxNgramDiff(int maxNgramDiff) { this.maxNgramDiff = maxNgramDiff; }
676+
677+
/**
678+
* Returns the maximum allowed difference between max and min shingle_size
679+
*/
680+
public int getMaxShingleDiff() { return this.maxShingleDiff; }
681+
682+
private void setMaxShingleDiff(int maxShingleDiff) { this.maxShingleDiff = maxShingleDiff; }
683+
644684
/**
645685
* Returns the maximum number of allowed script_fields to retrieve in a search request
646686
*/

‎core/src/main/java/org/elasticsearch/index/analysis/NGramTokenizerFactory.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.lucene.analysis.Tokenizer;
2323
import org.apache.lucene.analysis.ngram.NGramTokenizer;
24+
import org.elasticsearch.Version;
2425
import org.elasticsearch.common.settings.Settings;
2526
import org.elasticsearch.env.Environment;
2627
import org.elasticsearch.index.IndexSettings;
@@ -84,8 +85,21 @@ static CharMatcher parseTokenChars(List<String> characterClasses) {
8485

8586
public NGramTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
8687
super(indexSettings, name, settings);
88+
int maxAllowedNgramDiff = indexSettings.getMaxNgramDiff();
8789
this.minGram = settings.getAsInt("min_gram", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
8890
this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
91+
int ngramDiff = maxGram - minGram;
92+
if (ngramDiff > maxAllowedNgramDiff) {
93+
if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) {
94+
throw new IllegalArgumentException(
95+
"The difference between max_gram and min_gram in NGram Tokenizer must be less than or equal to: ["
96+
+ maxAllowedNgramDiff + "] but was [" + ngramDiff + "]. This limit can be set by changing the ["
97+
+ IndexSettings.MAX_NGRAM_DIFF_SETTING.getKey() + "] index level setting.");
98+
} else {
99+
deprecationLogger.deprecated("Deprecated big difference between max_gram and min_gram in NGram Tokenizer,"
100+
+ "expected difference must be less than or equal to: [" + maxAllowedNgramDiff + "]");
101+
}
102+
}
89103
this.matcher = parseTokenChars(settings.getAsList("token_chars"));
90104
}
91105

0 commit comments

Comments
 (0)