clusterBlocks = currentState.blocks().indices().get(index.getName());
+ if (clusterBlocks != null) {
+ for (ClusterBlock clusterBlock : clusterBlocks) {
+ if (clusterBlock.id() == INDEX_CLOSED_BLOCK_ID) {
+ // Reuse the existing index closed block
+ indexBlock = clusterBlock;
+ break;
+ }
+ }
+ }
+ if (useDirectClose) {
+ logger.debug("closing index {} directly", index);
+ metadata.put(IndexMetaData.builder(indexToClose).state(IndexMetaData.State.CLOSE)); // increment version?
+ blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID);
+ routingTable.remove(index.getName());
+ indexBlock = INDEX_CLOSED_BLOCK;
+ } else {
+ if (indexBlock == null) {
+ // Create a new index closed block
+ indexBlock = createIndexClosingBlock();
+ }
+ assert Strings.hasLength(indexBlock.uuid()) : "Closing block should have a UUID";
+ }
+ blocks.addIndexBlock(index.getName(), indexBlock);
+ blockedIndices.put(index, indexBlock);
}
- ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocksBuilder).build();
+ logger.info(() -> new ParameterizedMessage("closing indices {}",
+ blockedIndices.keySet().stream().map(Object::toString).collect(Collectors.joining(","))));
+ return ClusterState.builder(currentState).blocks(blocks).metaData(metadata).routingTable(routingTable.build()).build();
+ }
- RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
- for (IndexMetaData index : indicesToClose) {
- rtBuilder.remove(index.getIndex().getName());
+ /**
+ * Step 2 - Wait for indices to be ready for closing
+ *
+ * This step iterates over the indices previously blocked and sends a {@link TransportVerifyShardBeforeCloseAction} to each shard. If
+ * this action succeed then the shard is considered to be ready for closing. When all shards of a given index are ready for closing,
+ * the index is considered ready to be closed.
+ */
+ class WaitForClosedBlocksApplied extends AbstractRunnable {
+
+ private final Map blockedIndices;
+ private final CloseIndexClusterStateUpdateRequest request;
+ private final ActionListener> listener;
+
+ private WaitForClosedBlocksApplied(final Map blockedIndices,
+ final CloseIndexClusterStateUpdateRequest request,
+ final ActionListener> listener) {
+ if (blockedIndices == null || blockedIndices.isEmpty()) {
+ throw new IllegalArgumentException("Cannot wait for closed blocks to be applied, list of blocked indices is empty or null");
+ }
+ this.blockedIndices = blockedIndices;
+ this.request = request;
+ this.listener = listener;
+ }
+
+ @Override
+ public void onFailure(final Exception e) {
+ listener.onFailure(e);
}
- //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
- return allocationService.reroute(
- ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(),
- "indices closed [" + indicesAsString + "]");
+ @Override
+ protected void doRun() throws Exception {
+ final Map results = ConcurrentCollections.newConcurrentMap();
+ final CountDown countDown = new CountDown(blockedIndices.size());
+ final ClusterState state = clusterService.state();
+ blockedIndices.forEach((index, block) -> {
+ waitForShardsReadyForClosing(index, block, state, response -> {
+ results.put(index, response);
+ if (countDown.countDown()) {
+ listener.onResponse(unmodifiableMap(results));
+ }
+ });
+ });
+ }
+
+ private void waitForShardsReadyForClosing(final Index index, final ClusterBlock closingBlock,
+ final ClusterState state, final Consumer onResponse) {
+ final IndexMetaData indexMetaData = state.metaData().index(index);
+ if (indexMetaData == null) {
+ logger.debug("index {} has been blocked before closing and is now deleted, ignoring", index);
+ onResponse.accept(new AcknowledgedResponse(true));
+ return;
+ }
+ final IndexRoutingTable indexRoutingTable = state.routingTable().index(index);
+ if (indexRoutingTable == null || indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+ assert state.blocks().hasIndexBlock(index.getName(), INDEX_CLOSED_BLOCK);
+ logger.debug("index {} has been blocked before closing and is already closed, ignoring", index);
+ onResponse.accept(new AcknowledgedResponse(true));
+ return;
+ }
+
+ final ImmutableOpenIntMap shards = indexRoutingTable.getShards();
+ final AtomicArray results = new AtomicArray<>(shards.size());
+ final CountDown countDown = new CountDown(shards.size());
+
+ for (IntObjectCursor shard : shards) {
+ final IndexShardRoutingTable shardRoutingTable = shard.value;
+ final ShardId shardId = shardRoutingTable.shardId();
+ sendVerifyShardBeforeCloseRequest(shardRoutingTable, closingBlock, new NotifyOnceListener() {
+ @Override
+ public void innerOnResponse(final ReplicationResponse replicationResponse) {
+ ReplicationResponse.ShardInfo shardInfo = replicationResponse.getShardInfo();
+ results.setOnce(shardId.id(), new AcknowledgedResponse(shardInfo.getFailed() == 0));
+ processIfFinished();
+ }
+
+ @Override
+ public void innerOnFailure(final Exception e) {
+ results.setOnce(shardId.id(), new AcknowledgedResponse(false));
+ processIfFinished();
+ }
+
+ private void processIfFinished() {
+ if (countDown.countDown()) {
+ final boolean acknowledged = results.asList().stream().allMatch(AcknowledgedResponse::isAcknowledged);
+ onResponse.accept(new AcknowledgedResponse(acknowledged));
+ }
+ }
+ });
+ }
+ }
+
+ private void sendVerifyShardBeforeCloseRequest(final IndexShardRoutingTable shardRoutingTable,
+ final ClusterBlock closingBlock,
+ final ActionListener listener) {
+ final ShardId shardId = shardRoutingTable.shardId();
+ if (shardRoutingTable.primaryShard().unassigned()) {
+ logger.debug("primary shard {} is unassigned, ignoring", shardId);
+ final ReplicationResponse response = new ReplicationResponse();
+ response.setShardInfo(new ReplicationResponse.ShardInfo(shardRoutingTable.size(), shardRoutingTable.size()));
+ listener.onResponse(response);
+ return;
+ }
+ final TaskId parentTaskId = new TaskId(clusterService.localNode().getId(), request.taskId());
+ final TransportVerifyShardBeforeCloseAction.ShardRequest shardRequest =
+ new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, closingBlock, parentTaskId);
+ if (request.ackTimeout() != null) {
+ shardRequest.timeout(request.ackTimeout());
+ }
+ transportVerifyShardBeforeCloseAction.execute(shardRequest, listener);
+ }
+ }
+
+ /**
+ * Step 3 - Move index states from OPEN to CLOSE in cluster state for indices that are ready for closing.
+ */
+ static ClusterState closeRoutingTable(final ClusterState currentState,
+ final Map blockedIndices,
+ final Map results) {
+ final MetaData.Builder metadata = MetaData.builder(currentState.metaData());
+ final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
+ final RoutingTable.Builder routingTable = RoutingTable.builder(currentState.routingTable());
+
+ final Set closedIndices = new HashSet<>();
+ for (Map.Entry result : results.entrySet()) {
+ final Index index = result.getKey();
+ final boolean acknowledged = result.getValue().isAcknowledged();
+ try {
+ if (acknowledged == false) {
+ logger.debug("verification of shards before closing {} failed", index);
+ continue;
+ }
+ final IndexMetaData indexMetaData = metadata.getSafe(index);
+ if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+ logger.debug("verification of shards before closing {} succeeded but index is already closed", index);
+ assert currentState.blocks().hasIndexBlock(index.getName(), INDEX_CLOSED_BLOCK);
+ continue;
+ }
+ final ClusterBlock closingBlock = blockedIndices.get(index);
+ if (currentState.blocks().hasIndexBlock(index.getName(), closingBlock) == false) {
+ logger.debug("verification of shards before closing {} succeeded but block has been removed in the meantime", index);
+ continue;
+ }
+
+ logger.debug("closing index {} succeeded", index);
+ blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID).addIndexBlock(index.getName(), INDEX_CLOSED_BLOCK);
+ metadata.put(IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.CLOSE));
+ routingTable.remove(index.getName());
+ closedIndices.add(index.getName());
+ } catch (final IndexNotFoundException e) {
+ logger.debug("index {} has been deleted since it was blocked before closing, ignoring", index);
+ }
+ }
+ logger.info("completed closing of indices {}", closedIndices);
+ return ClusterState.builder(currentState).blocks(blocks).metaData(metadata).routingTable(routingTable.build()).build();
}
public void openIndex(final OpenIndexClusterStateUpdateRequest request,
@@ -170,64 +449,73 @@ private void onlyOpenIndex(final OpenIndexClusterStateUpdateRequest request,
final String indicesAsString = Arrays.toString(request.indices());
clusterService.submitStateUpdateTask("open-indices " + indicesAsString,
- new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) {
- @Override
- protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
- return new ClusterStateUpdateResponse(acknowledged);
- }
-
- @Override
- public ClusterState execute(ClusterState currentState) {
- List indicesToOpen = new ArrayList<>();
- for (Index index : request.indices()) {
- final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
- if (indexMetaData.getState() != IndexMetaData.State.OPEN) {
- indicesToOpen.add(indexMetaData);
- }
+ new AckedClusterStateUpdateTask(Priority.URGENT, request, listener) {
+ @Override
+ protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
+ return new ClusterStateUpdateResponse(acknowledged);
}
- validateShardLimit(currentState, request.indices());
-
- if (indicesToOpen.isEmpty()) {
- return currentState;
+ @Override
+ public ClusterState execute(final ClusterState currentState) {
+ final ClusterState updatedState = openIndices(request.indices(), currentState);
+ //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
+ return allocationService.reroute(updatedState, "indices opened [" + indicesAsString + "]");
}
+ }
+ );
+ }
- logger.info("opening indices [{}]", indicesAsString);
-
- MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
- ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder()
- .blocks(currentState.blocks());
- final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion()
- .minimumIndexCompatibilityVersion();
- for (IndexMetaData closedMetaData : indicesToOpen) {
- final String indexName = closedMetaData.getIndex().getName();
- IndexMetaData indexMetaData = IndexMetaData.builder(closedMetaData).state(IndexMetaData.State.OPEN).build();
- // The index might be closed because we couldn't import it due to old incompatible version
- // We need to check that this index can be upgraded to the current version
- indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData, minIndexCompatibilityVersion);
- try {
- indicesService.verifyIndexMetadata(indexMetaData, indexMetaData);
- } catch (Exception e) {
- throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e);
- }
+ ClusterState openIndices(final Index[] indices, final ClusterState currentState) {
+ final List indicesToOpen = new ArrayList<>();
+ for (Index index : indices) {
+ final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index);
+ if (indexMetaData.getState() != IndexMetaData.State.OPEN) {
+ indicesToOpen.add(indexMetaData);
+ } else if (currentState.blocks().hasIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID)) {
+ indicesToOpen.add(indexMetaData);
+ }
+ }
- mdBuilder.put(indexMetaData, true);
- blocksBuilder.removeIndexBlock(indexName, INDEX_CLOSED_BLOCK);
+ validateShardLimit(currentState, indices);
+ if (indicesToOpen.isEmpty()) {
+ return currentState;
+ }
+
+ logger.info(() -> new ParameterizedMessage("opening indices [{}]",
+ String.join(",", indicesToOpen.stream().map(i -> (CharSequence) i.getIndex().toString())::iterator)));
+
+ final MetaData.Builder metadata = MetaData.builder(currentState.metaData());
+ final ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
+ final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion();
+
+ for (IndexMetaData indexMetaData : indicesToOpen) {
+ final Index index = indexMetaData.getIndex();
+ if (indexMetaData.getState() != IndexMetaData.State.OPEN) {
+ IndexMetaData updatedIndexMetaData = IndexMetaData.builder(indexMetaData).state(IndexMetaData.State.OPEN).build();
+ // The index might be closed because we couldn't import it due to old incompatible version
+ // We need to check that this index can be upgraded to the current version
+ updatedIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(updatedIndexMetaData, minIndexCompatibilityVersion);
+ try {
+ indicesService.verifyIndexMetadata(updatedIndexMetaData, updatedIndexMetaData);
+ } catch (Exception e) {
+ throw new ElasticsearchException("Failed to verify index " + index, e);
}
+ metadata.put(updatedIndexMetaData, true);
+ }
- ClusterState updatedState = ClusterState.builder(currentState).metaData(mdBuilder).blocks(blocksBuilder).build();
+ // Always removes index closed blocks (note: this can fail on-going close index actions)
+ blocks.removeIndexBlockWithId(index.getName(), INDEX_CLOSED_BLOCK_ID);
+ }
- RoutingTable.Builder rtBuilder = RoutingTable.builder(updatedState.routingTable());
- for (IndexMetaData index : indicesToOpen) {
- rtBuilder.addAsFromCloseToOpen(updatedState.metaData().getIndexSafe(index.getIndex()));
- }
+ ClusterState updatedState = ClusterState.builder(currentState).metaData(metadata).blocks(blocks).build();
- //no explicit wait for other nodes needed as we use AckedClusterStateUpdateTask
- return allocationService.reroute(
- ClusterState.builder(updatedState).routingTable(rtBuilder.build()).build(),
- "indices opened [" + indicesAsString + "]");
+ final RoutingTable.Builder routingTable = RoutingTable.builder(updatedState.routingTable());
+ for (IndexMetaData previousIndexMetaData : indicesToOpen) {
+ if (previousIndexMetaData.getState() != IndexMetaData.State.OPEN) {
+ routingTable.addAsFromCloseToOpen(updatedState.metaData().getIndexSafe(previousIndexMetaData.getIndex()));
}
- });
+ }
+ return ClusterState.builder(updatedState).routingTable(routingTable.build()).build();
}
/**
@@ -250,7 +538,6 @@ static void validateShardLimit(ClusterState currentState, Index[] indices) {
ex.addValidationError(error.get());
throw ex;
}
-
}
private static int getTotalShardCount(ClusterState state, Index index) {
@@ -258,4 +545,14 @@ private static int getTotalShardCount(ClusterState state, Index index) {
return indexMetaData.getNumberOfShards() * (1 + indexMetaData.getNumberOfReplicas());
}
+ /**
+ * @return Generates a {@link ClusterBlock} that blocks read and write operations on soon-to-be-closed indices. The
+ * cluster block is generated with the id value equals to {@link #INDEX_CLOSED_BLOCK_ID} and a unique UUID.
+ */
+ public static ClusterBlock createIndexClosingBlock() {
+ return new ClusterBlock(INDEX_CLOSED_BLOCK_ID, UUIDs.randomBase64UUID(), "index preparing to close. Reopen the index to allow " +
+ "writes again or retry closing the index to fully close the index.", false, false, false, RestStatus.FORBIDDEN,
+ EnumSet.of(ClusterBlockLevel.WRITE));
+ }
+
}
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
index c331a9a137e0f..5bd441419d136 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java
@@ -89,7 +89,7 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements
private final Collection clusterStateListeners = new CopyOnWriteArrayList<>();
private final Collection timeoutClusterStateListeners =
- Collections.newSetFromMap(new ConcurrentHashMap());
+ Collections.newSetFromMap(new ConcurrentHashMap<>());
private final LocalNodeMasterListeners localNodeMasterListeners;
@@ -134,11 +134,15 @@ protected synchronized void doStart() {
Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
Objects.requireNonNull(state.get(), "please set initial state before starting");
addListener(localNodeMasterListeners);
- threadPoolExecutor = EsExecutors.newSinglePrioritizing(
- nodeName + "/" + CLUSTER_UPDATE_THREAD_NAME,
- daemonThreadFactory(nodeName, CLUSTER_UPDATE_THREAD_NAME),
- threadPool.getThreadContext(),
- threadPool.scheduler());
+ threadPoolExecutor = createThreadPoolExecutor();
+ }
+
+ protected PrioritizedEsThreadPoolExecutor createThreadPoolExecutor() {
+ return EsExecutors.newSinglePrioritizing(
+ nodeName + "/" + CLUSTER_UPDATE_THREAD_NAME,
+ daemonThreadFactory(nodeName, CLUSTER_UPDATE_THREAD_NAME),
+ threadPool.getThreadContext(),
+ threadPool.scheduler());
}
class UpdateTask extends SourcePrioritizedRunnable implements Function {
diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index fa4f6acc65d23..12d45c4fb88f4 100644
--- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -71,8 +71,13 @@ public class ClusterService extends AbstractLifecycleComponent {
private final String nodeName;
- public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool,
- MasterService masterService) {
+ public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+ this(settings, clusterSettings, new MasterService(Node.NODE_NAME_SETTING.get(settings), settings, threadPool),
+ new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool));
+ }
+
+ public ClusterService(Settings settings, ClusterSettings clusterSettings, MasterService masterService,
+ ClusterApplierService clusterApplierService) {
super(settings);
this.settings = settings;
this.nodeName = Node.NODE_NAME_SETTING.get(settings);
@@ -84,11 +89,7 @@ public ClusterService(Settings settings, ClusterSettings clusterSettings, Thread
this::setSlowTaskLoggingThreshold);
// Add a no-op update consumer so changes are logged
this.clusterSettings.addAffixUpdateConsumer(USER_DEFINED_META_DATA, (first, second) -> {}, (first, second) -> {});
- this.clusterApplierService = new ClusterApplierService(nodeName, settings, clusterSettings, threadPool);
- }
-
- public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
- this(settings, clusterSettings, threadPool, new MasterService(Node.NODE_NAME_SETTING.get(settings), settings, threadPool));
+ this.clusterApplierService = clusterApplierService;
}
private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
index 795cc235ce759..7079bcc5b4606 100644
--- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
+++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
@@ -32,7 +32,6 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.FieldData;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
@@ -555,23 +554,26 @@ private static GeoPoint parseGeoHash(GeoPoint point, String geohash, EffectivePo
* @return int representing precision
*/
public static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException {
- XContentParser.Token token = parser.currentToken();
- if (token.equals(XContentParser.Token.VALUE_NUMBER)) {
- return XContentMapValues.nodeIntegerValue(parser.intValue());
- } else {
- String precision = parser.text();
+ return parser.currentToken() == Token.VALUE_NUMBER ? parser.intValue() : parsePrecisionString(parser.text());
+ }
+
+ /**
+ * Attempt to parse geohash precision string into an integer value
+ */
+ public static int parsePrecisionString(String precision) {
+ try {
+ // we want to treat simple integer strings as precision levels, not distances
+ return checkPrecisionRange(Integer.parseInt(precision));
+ // checkPrecisionRange could also throw IllegalArgumentException, but let it through
+ // to keep errors somewhat consistent with how they were shown before this change
+ } catch (NumberFormatException e) {
+ // try to parse as a distance value
+ final int parsedPrecision = GeoUtils.geoHashLevelsForPrecision(precision);
try {
- // we want to treat simple integer strings as precision levels, not distances
- return XContentMapValues.nodeIntegerValue(precision);
- } catch (NumberFormatException e) {
- // try to parse as a distance value
- final int parsedPrecision = GeoUtils.geoHashLevelsForPrecision(precision);
- try {
- return checkPrecisionRange(parsedPrecision);
- } catch (IllegalArgumentException e2) {
- // this happens when distance too small, so precision > 12. We'd like to see the original string
- throw new IllegalArgumentException("precision too high [" + precision + "]", e2);
- }
+ return checkPrecisionRange(parsedPrecision);
+ } catch (IllegalArgumentException e2) {
+ // this happens when distance too small, so precision > 12. We'd like to see the original string
+ throw new IllegalArgumentException("precision too high [" + precision + "]", e2);
}
}
}
diff --git a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index 55e2a1836fda3..165256940bb81 100644
--- a/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/server/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -20,6 +20,7 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.elasticsearch.Version;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -130,7 +131,13 @@ public Analyzer getAnalyzer(String analyzer) throws IOException {
throw new ElasticsearchException("failed to load analyzer for name " + key, ex);
}}
);
+ } else if ("standard_html_strip".equals(analyzer)) {
+ if (Version.CURRENT.onOrAfter(Version.V_7_0_0)) {
+ throw new IllegalArgumentException("[standard_html_strip] analyzer is not supported for new indices, " +
+ "use a custom analyzer using [standard] tokenizer and [html_strip] char_filter, plus [lowercase] filter");
+ }
}
+
return analyzerProvider.get(environment, analyzer).get();
}
diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 6ee0f3f10ddcc..f8ffcfdc05bcc 100644
--- a/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
+import org.apache.logging.log4j.LogManager;
import org.apache.lucene.document.LatLonShape;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
@@ -38,6 +39,7 @@
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.geo.GeoPoint;
@@ -48,6 +50,7 @@
import org.elasticsearch.common.geo.parsers.ShapeParser;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -66,6 +69,10 @@
*/
public class GeoShapeQueryBuilder extends AbstractQueryBuilder {
public static final String NAME = "geo_shape";
+ private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+ LogManager.getLogger(GeoShapeQueryBuilder.class));
+ static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [geo_shape] queries. " +
+ "The type should no longer be specified in the [indexed_shape] section.";
public static final String DEFAULT_SHAPE_INDEX_NAME = "shapes";
public static final String DEFAULT_SHAPE_FIELD_NAME = "shape";
@@ -119,6 +126,19 @@ public GeoShapeQueryBuilder(String fieldName, ShapeBuilder shape) {
this(fieldName, shape, null, null);
}
+ /**
+ * Creates a new GeoShapeQueryBuilder whose Query will be against the given
+ * field name and will use the Shape found with the given ID
+ *
+ * @param fieldName
+ * Name of the field that will be filtered
+ * @param indexedShapeId
+ * ID of the indexed Shape that will be used in the Query
+ */
+ public GeoShapeQueryBuilder(String fieldName, String indexedShapeId) {
+ this(fieldName, (ShapeBuilder) null, indexedShapeId, null);
+ }
+
/**
* Creates a new GeoShapeQueryBuilder whose Query will be against the given
* field name and will use the Shape found with the given ID in the given
@@ -130,20 +150,19 @@ public GeoShapeQueryBuilder(String fieldName, ShapeBuilder shape) {
* ID of the indexed Shape that will be used in the Query
* @param indexedShapeType
* Index type of the indexed Shapes
+ * @deprecated use {@link #GeoShapeQueryBuilder(String, String)} instead
*/
+ @Deprecated
public GeoShapeQueryBuilder(String fieldName, String indexedShapeId, String indexedShapeType) {
this(fieldName, (ShapeBuilder) null, indexedShapeId, indexedShapeType);
}
- private GeoShapeQueryBuilder(String fieldName, ShapeBuilder shape, String indexedShapeId, String indexedShapeType) {
+ private GeoShapeQueryBuilder(String fieldName, ShapeBuilder shape, String indexedShapeId, @Nullable String indexedShapeType) {
if (fieldName == null) {
throw new IllegalArgumentException("fieldName is required");
}
if (shape == null && indexedShapeId == null) {
- throw new IllegalArgumentException("either shapeBytes or indexedShapeId and indexedShapeType are required");
- }
- if (indexedShapeId != null && indexedShapeType == null) {
- throw new IllegalArgumentException("indexedShapeType is required if indexedShapeId is specified");
+ throw new IllegalArgumentException("either shape or indexedShapeId is required");
}
this.fieldName = fieldName;
this.shape = shape;
@@ -152,7 +171,8 @@ private GeoShapeQueryBuilder(String fieldName, ShapeBuilder shape, String indexe
this.supplier = null;
}
- private GeoShapeQueryBuilder(String fieldName, Supplier supplier, String indexedShapeId, String indexedShapeType) {
+ private GeoShapeQueryBuilder(String fieldName, Supplier supplier, String indexedShapeId,
+ @Nullable String indexedShapeType) {
this.fieldName = fieldName;
this.shape = null;
this.supplier = supplier;
@@ -238,7 +258,10 @@ public String indexedShapeId() {
/**
* @return the document type of the indexed Shape that will be used in the
* Query
+ *
+ * @deprecated Types are in the process of being removed.
*/
+ @Deprecated
public String indexedShapeType() {
return indexedShapeType;
}
@@ -566,8 +589,10 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
shape.toXContent(builder, params);
} else {
builder.startObject(INDEXED_SHAPE_FIELD.getPreferredName())
- .field(SHAPE_ID_FIELD.getPreferredName(), indexedShapeId)
- .field(SHAPE_TYPE_FIELD.getPreferredName(), indexedShapeType);
+ .field(SHAPE_ID_FIELD.getPreferredName(), indexedShapeId);
+ if (indexedShapeType != null) {
+ builder.field(SHAPE_TYPE_FIELD.getPreferredName(), indexedShapeType);
+ }
if (indexedShapeIndex != null) {
builder.field(SHAPE_INDEX_FIELD.getPreferredName(), indexedShapeIndex);
}
@@ -677,6 +702,11 @@ public static GeoShapeQueryBuilder fromXContent(XContentParser parser) throws IO
}
}
GeoShapeQueryBuilder builder;
+ if (type != null) {
+ deprecationLogger.deprecatedAndMaybeLog(
+ "geo_share_query_with_types", TYPES_DEPRECATION_MESSAGE);
+ }
+
if (shape != null) {
builder = new GeoShapeQueryBuilder(fieldName, shape);
} else {
@@ -739,7 +769,12 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
} else if (this.shape == null) {
SetOnce supplier = new SetOnce<>();
queryRewriteContext.registerAsyncAction((client, listener) -> {
- GetRequest getRequest = new GetRequest(indexedShapeIndex, indexedShapeType, indexedShapeId);
+ GetRequest getRequest;
+ if (indexedShapeType == null) {
+ getRequest = new GetRequest(indexedShapeIndex, indexedShapeId);
+ } else {
+ getRequest = new GetRequest(indexedShapeIndex, indexedShapeType, indexedShapeId);
+ }
getRequest.routing(indexedShapeRouting);
fetch(client, getRequest, indexedShapePath, ActionListener.wrap(builder-> {
supplier.set(builder);
diff --git a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
index 7cbd38f3398fd..358a2fccff108 100644
--- a/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
@@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
+import org.apache.logging.log4j.LogManager;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.cluster.metadata.MetaData;
@@ -27,6 +28,7 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -51,6 +53,9 @@
*/
public class IdsQueryBuilder extends AbstractQueryBuilder {
public static final String NAME = "ids";
+ private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+ LogManager.getLogger(IdsQueryBuilder.class));
+ static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [ids] queries.";
private static final ParseField TYPE_FIELD = new ParseField("type");
private static final ParseField VALUES_FIELD = new ParseField("values");
@@ -83,8 +88,10 @@ protected void doWriteTo(StreamOutput out) throws IOException {
/**
* Add types to query
+ *
+ * @deprecated Types are in the process of being removed, prefer to filter on a field instead.
*/
- // TODO: Remove
+ @Deprecated
public IdsQueryBuilder types(String... types) {
if (types == null) {
throw new IllegalArgumentException("[" + NAME + "] types cannot be null");
@@ -95,7 +102,10 @@ public IdsQueryBuilder types(String... types) {
/**
* Returns the types used in this query
+ *
+ * @deprecated Types are in the process of being removed, prefer to filter on a field instead.
*/
+ @Deprecated
public String[] types() {
return this.types;
}
@@ -121,7 +131,9 @@ public Set ids() {
@Override
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(NAME);
- builder.array(TYPE_FIELD.getPreferredName(), types);
+ if (types.length > 0) {
+ builder.array(TYPE_FIELD.getPreferredName(), types);
+ }
builder.startArray(VALUES_FIELD.getPreferredName());
for (String value : ids) {
builder.value(value);
@@ -142,7 +154,11 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
public static IdsQueryBuilder fromXContent(XContentParser parser) {
try {
- return PARSER.apply(parser, null);
+ IdsQueryBuilder builder = PARSER.apply(parser, null);
+ if (builder.types().length > 0) {
+ deprecationLogger.deprecatedAndMaybeLog("ids_query_with_types", TYPES_DEPRECATION_MESSAGE);
+ }
+ return builder;
} catch (IllegalArgumentException e) {
throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e);
}
diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 6f87ac0fda79b..b90a1e60ffa0b 100644
--- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
+import org.apache.logging.log4j.LogManager;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Fields;
import org.apache.lucene.search.BooleanClause;
@@ -41,6 +42,7 @@
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.search.MoreLikeThisQuery;
import org.elasticsearch.common.lucene.search.XMoreLikeThis;
import org.elasticsearch.common.lucene.uid.Versions;
@@ -53,6 +55,7 @@
import org.elasticsearch.index.mapper.IdFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
import java.io.IOException;
@@ -66,6 +69,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Stream;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -76,6 +80,11 @@
*/
public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder {
public static final String NAME = "more_like_this";
+ private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+ LogManager.getLogger(MoreLikeThisQueryBuilder.class));
+ static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [more_like_this] " +
+ "queries. The type should no longer be specified in the [like] and [unlike] sections.";
+
public static final int DEFAULT_MAX_QUERY_TERMS = XMoreLikeThis.DEFAULT_MAX_QUERY_TERMS;
public static final int DEFAULT_MIN_TERM_FREQ = XMoreLikeThis.DEFAULT_MIN_TERM_FREQ;
@@ -178,13 +187,45 @@ public Item() {
this.versionType = copy.versionType;
}
+ /**
+ * Constructor for a given item / document request
+ *
+ * @param index the index where the document is located
+ * @param id and its id
+ */
+ public Item(@Nullable String index, String id) {
+ if (id == null) {
+ throw new IllegalArgumentException("Item requires id to be non-null");
+ }
+ this.index = index;
+ this.id = id;
+ }
+
+ /**
+ * Constructor for an artificial document request, that is not present in the index.
+ *
+ * @param index the index to be used for parsing the doc
+ * @param doc the document specification
+ */
+ public Item(@Nullable String index, XContentBuilder doc) {
+ if (doc == null) {
+ throw new IllegalArgumentException("Item requires doc to be non-null");
+ }
+ this.index = index;
+ this.doc = BytesReference.bytes(doc);
+ this.xContentType = doc.contentType();
+ }
+
/**
* Constructor for a given item / document request
*
* @param index the index where the document is located
* @param type the type of the document
* @param id and its id
+ *
+ * @deprecated Types are in the process of being removed, use {@link Item(String, String)} instead.
*/
+ @Deprecated
public Item(@Nullable String index, @Nullable String type, String id) {
if (id == null) {
throw new IllegalArgumentException("Item requires id to be non-null");
@@ -200,7 +241,10 @@ public Item(@Nullable String index, @Nullable String type, String id) {
* @param index the index to be used for parsing the doc
* @param type the type to be used for parsing the doc
* @param doc the document specification
+ *
+ * @deprecated Types are in the process of being removed, use {@link Item(String, XContentBuilder)} instead.
*/
+ @Deprecated
public Item(@Nullable String index, @Nullable String type, XContentBuilder doc) {
if (doc == null) {
throw new IllegalArgumentException("Item requires doc to be non-null");
@@ -257,10 +301,18 @@ public Item index(String index) {
return this;
}
+ /**
+ * @deprecated Types are in the process of being removed.
+ */
+ @Deprecated
public String type() {
return type;
}
+ /**
+ * @deprecated Types are in the process of being removed.
+ */
+ @Deprecated
public Item type(String type) {
this.type = type;
return this;
@@ -912,9 +964,18 @@ public static MoreLikeThisQueryBuilder fromXContent(XContentParser parser) throw
if (stopWords != null) {
moreLikeThisQueryBuilder.stopWords(stopWords);
}
+
+ if (moreLikeThisQueryBuilder.isTypeless() == false) {
+ deprecationLogger.deprecatedAndMaybeLog("more_like_this_query_with_types", TYPES_DEPRECATION_MESSAGE);
+ }
return moreLikeThisQueryBuilder;
}
+ public boolean isTypeless() {
+ return Stream.concat(Arrays.stream(likeItems), Arrays.stream(unlikeItems))
+ .allMatch(item -> item.type == null);
+ }
+
private static void parseLikeField(XContentParser parser, List texts, List- items) throws IOException {
if (parser.currentToken().isValue()) {
texts.add(parser.text());
@@ -1065,12 +1126,7 @@ private static void setDefaultIndexTypeFields(QueryShardContext context, Item it
item.index(context.index().getName());
}
if (item.type() == null) {
- if (context.queryTypes().size() > 1) {
- throw new QueryShardException(context,
- "ambiguous type for item with id: " + item.id() + " and index: " + item.index());
- } else {
- item.type(context.queryTypes().iterator().next());
- }
+ item.type(MapperService.SINGLE_MAPPING_NAME);
}
// default fields if not present but don't override for artificial docs
if ((item.fields() == null || item.fields().length == 0) && item.doc() == null) {
diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index f5cf2d5da66be..5ac70781286a4 100644
--- a/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/server/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -121,7 +121,10 @@ public static IdsQueryBuilder idsQuery() {
* Constructs a query that will match only specific ids within types.
*
* @param types The mapping/doc type
+ *
+ * @deprecated Types are in the process of being removed, use {@link #idsQuery()} instead.
*/
+ @Deprecated
public static IdsQueryBuilder idsQuery(String... types) {
return new IdsQueryBuilder().types(types);
}
@@ -646,6 +649,14 @@ public static GeoShapeQueryBuilder geoShapeQuery(String name, ShapeBuilder shape
return new GeoShapeQueryBuilder(name, shape);
}
+ public static GeoShapeQueryBuilder geoShapeQuery(String name, String indexedShapeId) {
+ return new GeoShapeQueryBuilder(name, indexedShapeId);
+ }
+
+ /**
+ * @deprecated Types are in the process of being removed, use {@link #geoShapeQuery(String, String)} instead.
+ */
+ @Deprecated
public static GeoShapeQueryBuilder geoShapeQuery(String name, String indexedShapeId, String indexedShapeType) {
return new GeoShapeQueryBuilder(name, indexedShapeId, indexedShapeType);
}
@@ -662,6 +673,16 @@ public static GeoShapeQueryBuilder geoIntersectionQuery(String name, ShapeBuilde
return builder;
}
+ public static GeoShapeQueryBuilder geoIntersectionQuery(String name, String indexedShapeId) {
+ GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId);
+ builder.relation(ShapeRelation.INTERSECTS);
+ return builder;
+ }
+
+ /**
+ * @deprecated Types are in the process of being removed, use {@link #geoIntersectionQuery(String, String)} instead.
+ */
+ @Deprecated
public static GeoShapeQueryBuilder geoIntersectionQuery(String name, String indexedShapeId, String indexedShapeType) {
GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId, indexedShapeType);
builder.relation(ShapeRelation.INTERSECTS);
@@ -680,6 +701,16 @@ public static GeoShapeQueryBuilder geoWithinQuery(String name, ShapeBuilder shap
return builder;
}
+ public static GeoShapeQueryBuilder geoWithinQuery(String name, String indexedShapeId) {
+ GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId);
+ builder.relation(ShapeRelation.WITHIN);
+ return builder;
+ }
+
+ /**
+ * @deprecated Types are in the process of being removed, use {@link #geoWithinQuery(String, String)} instead.
+ */
+ @Deprecated
public static GeoShapeQueryBuilder geoWithinQuery(String name, String indexedShapeId, String indexedShapeType) {
GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId, indexedShapeType);
builder.relation(ShapeRelation.WITHIN);
@@ -698,6 +729,16 @@ public static GeoShapeQueryBuilder geoDisjointQuery(String name, ShapeBuilder sh
return builder;
}
+ public static GeoShapeQueryBuilder geoDisjointQuery(String name, String indexedShapeId) {
+ GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId);
+ builder.relation(ShapeRelation.DISJOINT);
+ return builder;
+ }
+
+ /**
+ * @deprecated Types are in the process of being removed, use {@link #geoDisjointQuery(String, String)} instead.
+ */
+ @Deprecated
public static GeoShapeQueryBuilder geoDisjointQuery(String name, String indexedShapeId, String indexedShapeType) {
GeoShapeQueryBuilder builder = geoShapeQuery(name, indexedShapeId, indexedShapeType);
builder.relation(ShapeRelation.DISJOINT);
diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index a144cbf088a01..ae7bbae63018b 100644
--- a/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/server/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
+import org.apache.logging.log4j.LogManager;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.util.BytesRef;
@@ -34,6 +35,7 @@
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -63,6 +65,11 @@
public class TermsQueryBuilder extends AbstractQueryBuilder {
public static final String NAME = "terms";
+ private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
+ LogManager.getLogger(TermsQueryBuilder.class));
+ static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Types are deprecated " +
+ "in [terms] lookup queries.";
+
private final String fieldName;
private final List values;
private final TermsLookup termsLookup;
@@ -211,6 +218,10 @@ public TermsLookup termsLookup() {
return this.termsLookup;
}
+ public boolean isTypeless() {
+ return termsLookup == null || termsLookup.type() == null;
+ }
+
private static final Set> INTEGER_TYPES = new HashSet<>(
Arrays.asList(Byte.class, Short.class, Integer.class, Long.class));
private static final Set> STRING_TYPES = new HashSet<>(
@@ -391,9 +402,16 @@ public static TermsQueryBuilder fromXContent(XContentParser parser) throws IOExc
throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query requires a field name, " +
"followed by array of terms or a document lookup specification");
}
- return new TermsQueryBuilder(fieldName, values, termsLookup)
- .boost(boost)
- .queryName(queryName);
+
+ TermsQueryBuilder builder = new TermsQueryBuilder(fieldName, values, termsLookup)
+ .boost(boost)
+ .queryName(queryName);
+
+ if (builder.isTypeless() == false) {
+ deprecationLogger.deprecatedAndMaybeLog("terms_lookup_with_types", TYPES_DEPRECATION_MESSAGE);
+ }
+
+ return builder;
}
static List