From fe71b5f16d48e809c232162cb87cedb627073666 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 17 Jul 2017 10:21:48 -0700 Subject: [PATCH 01/21] wip #24547 --- .../reindex/AbstractBulkByScrollRequest.java | 24 ++-- .../index/reindex/SlicesCount.java | 124 ++++++++++++++++++ 2 files changed, 135 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 9f10304622ba4..c4c17c46eae7e 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -102,7 +102,7 @@ public abstract class AbstractBulkByScrollRequest 1) { return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId, slices); } @@ -409,9 +407,9 @@ public void readFrom(StreamInput in) throws IOException { maxRetries = in.readVInt(); requestsPerSecond = in.readFloat(); if (in.getVersion().onOrAfter(Version.V_5_1_1)) { - slices = in.readVInt(); + slices = new SlicesCount(in); } else { - slices = 1; + slices = SlicesCount.DEFAULT; } } @@ -428,9 +426,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(maxRetries); out.writeFloat(requestsPerSecond); if (out.getVersion().onOrAfter(Version.V_5_1_1)) { - out.writeVInt(slices); + slices.writeTo(out); } else { - if (slices > 1) { + if (slices.isAuto() || slices.number() > 1) { throw new IllegalArgumentException("Attempting to send sliced reindex-style request to a node that doesn't support " + "it. Version is [" + out.getVersion() + "] but must be [" + Version.V_5_1_1 + "]"); } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java new file mode 100644 index 0000000000000..9a2fb19a7da20 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public final class SlicesCount implements ToXContent, Writeable { + + private static final int AUTO_COUNT = -1; + + public static final String FIELD_NAME = "slices"; + public static final String AUTO_VALUE = "auto"; + + public static final SlicesCount AUTO = new SlicesCount(AUTO_COUNT); + public static final SlicesCount ONE = fromNumber(1); + public static final SlicesCount DEFAULT = ONE; + + private final int count; + + private SlicesCount(int count) { + this.count = count; + } + + public SlicesCount(StreamInput stream) throws IOException { + count = stream.readVInt(); + } + + public static SlicesCount fromNumber(int count) { + if (count < 1) { + throw new IllegalArgumentException("Slice count must be at least 1"); + } + return new SlicesCount(count); + } + + public static SlicesCount parse(String slicesString) { + Objects.requireNonNull(slicesString); + + if (AUTO_VALUE.equals(slicesString)) { + return AUTO; + } + + try { + int slicesNumber = Integer.parseInt(slicesString); + return fromNumber(slicesNumber); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Slices must be a positive integer or the string \"auto\""); + } + } + + public boolean isAuto() { + return count == AUTO_COUNT; + } + + public boolean isNumber() { + return !isAuto(); + } + + public int number() { + if (isAuto()) { + throw new IllegalStateException("Slice count is set as \"auto\", not a number"); + } + return count; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(count); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (isAuto()) { + builder.field(FIELD_NAME, AUTO_VALUE); + } else { + builder.field(FIELD_NAME, number()); + } + return builder; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + + SlicesCount other = (SlicesCount) obj; + return other.count == count; + } + + @Override + public int hashCode() { + return Objects.hash(count); + } +} From 87a4d140babe8da31fa4521eac408c18e7b1e79a Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 17 Jul 2017 20:15:58 -0700 Subject: [PATCH 02/21] wip #24547 Just uses the total group size as the slice count All unit and integ tests that were modified pass Module rest tests also pass with new tests for auto setting --- .../reindex/AbstractBulkByScrollRequest.java | 13 ++-- .../AbstractBulkByScrollRequestBuilder.java | 4 +- .../AbstractBulkIndexByScrollRequest.java | 4 +- .../index/reindex/DeleteByQueryRequest.java | 4 +- .../index/reindex/ParentBulkByScrollTask.java | 27 +++++-- .../index/reindex/ReindexRequest.java | 6 +- .../index/reindex/SlicesCount.java | 6 +- .../index/reindex/UpdateByQueryRequest.java | 4 +- .../AbstractBaseReindexRestHandler.java | 6 +- .../BulkByScrollParallelizationHelper.java | 6 +- .../reindex/TransportDeleteByQueryAction.java | 18 ++++- .../index/reindex/TransportReindexAction.java | 20 ++++- .../reindex/TransportUpdateByQueryAction.java | 17 ++++- .../reindex/AsyncBulkByScrollActionTests.java | 2 +- .../index/reindex/CancelTests.java | 22 +++--- .../reindex/DeleteByQueryBasicTests.java | 5 +- .../index/reindex/ReindexBasicTests.java | 4 +- .../index/reindex/RethrottleTests.java | 22 +++--- .../index/reindex/RoundTripTests.java | 14 ++-- .../TransportRethrottleActionTests.java | 3 +- .../reindex/UpdateByQueryBasicTests.java | 6 +- .../test/delete_by_query/80_slices.yml | 74 +++++++++++++++++++ .../rest-api-spec/test/reindex/80_slices.yml | 69 +++++++++++++++++ .../test/update_by_query/70_slices.yml | 67 +++++++++++++++++ 24 files changed, 348 insertions(+), 75 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index c4c17c46eae7e..32cfe9f03bbf6 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -355,24 +355,24 @@ public SlicesCount getSlices() { /** * Build a new request for a slice of the parent request. */ - public abstract Self forSlice(TaskId slicingTask, SearchRequest slice); + public abstract Self forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices); /** * Setup a clone of this request with the information needed to process a slice of it. */ - protected Self doForSlice(Self request, TaskId slicingTask) { + protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) { request.setAbortOnVersionConflict(abortOnVersionConflict).setRefresh(refresh).setTimeout(timeout) .setWaitForActiveShards(activeShardCount).setRetryBackoffInitialTime(retryBackoffInitialTime).setMaxRetries(maxRetries) // Parent task will store result .setShouldStoreResult(false) // Split requests per second between all slices - .setRequestsPerSecond(requestsPerSecond / slices) //todo need to have computed the actual thing here + .setRequestsPerSecond(requestsPerSecond / totalSlices) //todo need to have computed the actual thing here // Sub requests don't have workers .setSlices(SlicesCount.ONE); if (size != -1) { // Size is split between workers. This means the size might round // down! - request.setSize(size == SIZE_ALL_MATCHES ? SIZE_ALL_MATCHES : size / slices); + request.setSize(size == SIZE_ALL_MATCHES ? SIZE_ALL_MATCHES : size / totalSlices); } // Set the parent task so this task is cancelled if we cancel the parent request.setParentTask(slicingTask); @@ -382,9 +382,8 @@ protected Self doForSlice(Self request, TaskId slicingTask) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { - // todo need to know the actual number here - if (slices > 1) { - return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId, slices); + if (slices.isAuto() || slices.number() > 1) { + return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId); } /* Extract the slice from the search request so it'll be available in the status. This is potentially useful for users that manually * slice their search requests so they can keep track of it and **absolutely** useful for automatically sliced reindex requests so diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java index de3f22f0943a8..dc7a394b58bc6 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java @@ -145,8 +145,8 @@ public Self setShouldStoreResult(boolean shouldStoreResult) { /** * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. */ - public Self setSlices(int workers) { - request.setSlices(workers); + public Self setSlices(SlicesCount slices) { + request.setSlices(slices); return self(); } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java index 62c2635b301c5..65cdcf52b6fff 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java @@ -68,8 +68,8 @@ public Self setScript(@Nullable Script script) { } @Override - protected Self doForSlice(Self request, TaskId slicingTask) { - return super.doForSlice(request, slicingTask).setScript(script); + protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) { + return super.doForSlice(request, slicingTask, totalSlices).setScript(script); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java index ad70748f3e431..20f87e047b6a3 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java @@ -81,8 +81,8 @@ public ActionRequestValidationException validate() { } @Override - public DeleteByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) { - return doForSlice(new DeleteByQueryRequest(slice, false), slicingTask); + public DeleteByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices) { + return doForSlice(new DeleteByQueryRequest(slice, false), slicingTask, totalSlices); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java index bea9bc203653d..77db6269d0ff6 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java @@ -40,13 +40,30 @@ public class ParentBulkByScrollTask extends BulkByScrollTask { * Holds the responses as they come back. This uses {@link Tuple} as an "Either" style holder where only the response or the exception * is set. */ - private final AtomicArray results; - private final AtomicInteger counter; + private AtomicArray results; + private AtomicInteger counter; - public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId, int slices) { + // todo it may still make sense to store the original slices value in here even if it's not used + // todo either enforce slices being set before calling task methods or separate that out into another class + // todo probably want to look at the tests for this class and make sure the slices set behavior is enforced + + public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { super(id, type, action, description, parentTaskId); - this.results = new AtomicArray<>(slices); - this.counter = new AtomicInteger(slices); + } + + public void setSlices(int slices) { + if (slices < 1) { + throw new IllegalArgumentException("Slices must be at least one"); + } + if (isSlicesSet()) { + throw new IllegalStateException("Slices are already set"); + } + results = new AtomicArray<>(slices); + counter = new AtomicInteger(slices); + } + + public boolean isSlicesSet() { + return results != null && counter != null; } @Override diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index 76944c7b80438..29ccb79d03085 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -94,7 +94,7 @@ public ActionRequestValidationException validate() { if (getSearchRequest().source().query() != null) { e = addValidationError("reindex from remote sources should use RemoteInfo's query instead of source's query", e); } - if (getSlices() != 1) { + if (getSlices().isAuto() || getSlices().number() > 1) { e = addValidationError("reindex from remote sources doesn't support workers > 1 but was [" + getSlices() + "]", e); } } @@ -127,8 +127,8 @@ public RemoteInfo getRemoteInfo() { } @Override - public ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice) { - ReindexRequest sliced = doForSlice(new ReindexRequest(slice, destination, false), slicingTask); + public ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices) { + ReindexRequest sliced = doForSlice(new ReindexRequest(slice, destination, false), slicingTask, totalSlices); sliced.setRemoteInfo(remoteInfo); return sliced; } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java index 9a2fb19a7da20..d45e91c9b6fad 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java @@ -36,7 +36,7 @@ public final class SlicesCount implements ToXContent, Writeable { public static final String AUTO_VALUE = "auto"; public static final SlicesCount AUTO = new SlicesCount(AUTO_COUNT); - public static final SlicesCount ONE = fromNumber(1); + public static final SlicesCount ONE = of(1); public static final SlicesCount DEFAULT = ONE; private final int count; @@ -49,7 +49,7 @@ public SlicesCount(StreamInput stream) throws IOException { count = stream.readVInt(); } - public static SlicesCount fromNumber(int count) { + public static SlicesCount of(int count) { if (count < 1) { throw new IllegalArgumentException("Slice count must be at least 1"); } @@ -65,7 +65,7 @@ public static SlicesCount parse(String slicesString) { try { int slicesNumber = Integer.parseInt(slicesString); - return fromNumber(slicesNumber); + return of(slicesNumber); } catch (NumberFormatException e) { throw new IllegalArgumentException("Slices must be a positive integer or the string \"auto\""); } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java index ad0123d76cedf..2be0e46dbcedf 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java @@ -71,8 +71,8 @@ protected UpdateByQueryRequest self() { } @Override - public UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) { - UpdateByQueryRequest request = doForSlice(new UpdateByQueryRequest(slice, false), slicingTask); + public UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices) { + UpdateByQueryRequest request = doForSlice(new UpdateByQueryRequest(slice, false), slicingTask, totalSlices); request.setPipeline(pipeline); return request; } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java index 64b02c4be8150..d12508ff065f1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -90,7 +90,11 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) { request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh())); request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout())); - request.setSlices(restRequest.paramAsInt("slices", request.getSlices())); + + String slices = restRequest.param(SlicesCount.FIELD_NAME); + if (slices != null) { + request.setSlices(SlicesCount.parse(slices)); + } String waitForActiveShards = restRequest.param("wait_for_active_shards"); if (waitForActiveShards != null) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 48f1030645450..2208127f3b391 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -38,11 +38,12 @@ private BulkByScrollParallelizationHelper() {} public static > void startSlices(Client client, TaskManager taskManager, Action action, String localNodeId, ParentBulkByScrollTask task, Request request, + int totalSlices, ActionListener listener) { TaskId parentTaskId = new TaskId(localNodeId, task.getId()); - for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, request.getSlices())) { + for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, totalSlices)) { // TODO move the request to the correct node. maybe here or somehow do it as part of startup for reindex in general.... - Request requestForSlice = request.forSlice(parentTaskId, slice); + Request requestForSlice = request.forSlice(parentTaskId, slice, totalSlices); ActionListener sliceListener = ActionListener.wrap( r -> task.onSliceResponse(listener, slice.source().slice().getId(), r), e -> task.onSliceFailure(listener, slice.source().slice().getId(), e)); @@ -80,5 +81,4 @@ static SearchRequest[] sliceIntoSubRequests(SearchRequest request, String field, } return slices; } - } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 99e1a9f166dd8..5ee5760ccc10f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -51,9 +51,23 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { - if (request.getSlices() > 1) { + + if (request.getSlices().isAuto()) { + client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( + r -> doDeleteByQuery(task, request, listener, r.getGroups().length), + e -> listener.onFailure(e) + )); + } else { + doDeleteByQuery(task, request, listener, request.getSlices().number()); + } + } + + private void doDeleteByQuery(Task task, DeleteByQueryRequest request, ActionListener listener, int slices) { + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, - clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener); + clusterService.localNode().getId(), parentTask, request, slices, listener); } else { ClusterState state = clusterService.state(); ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 737d885443a1a..6c5a9ecfdecfa 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse.Failure; import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; @@ -109,9 +110,24 @@ public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFi @Override protected void doExecute(Task task, ReindexRequest request, ActionListener listener) { - if (request.getSlices() > 1) { + if (request.getSlices().isAuto()) { + // we need to look at shard details to decide on a slice count + client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( + r -> doReindex(task, request, listener, r.getGroups().length), + e -> listener.onFailure(e) + )); + } else { + // we already know an explicit slice count + doReindex(task, request, listener, request.getSlices().number()); + } + } + + private void doReindex(Task task, ReindexRequest request, ActionListener listener, int slices) { + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, clusterService.localNode().getId(), - (ParentBulkByScrollTask) task, request, listener); + parentTask, request, slices, listener); } else { checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); ClusterState state = clusterService.state(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 8924c7038c99e..16d09ab1feffe 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -64,9 +64,22 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { - if (request.getSlices() > 1) { + if (request.getSlices().isAuto()) { + client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( + r -> doUpdateByQuery(task, request, listener, r.getGroups().length), + e -> listener.onFailure(e) + )); + } else { + doUpdateByQuery(task, request, listener, request.getSlices().number()); + } + } + + private void doUpdateByQuery(Task task, UpdateByQueryRequest request, ActionListener listener, int slices) { + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, - clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener); + clusterService.localNode().getId(), parentTask, request, slices, listener); } else { ClusterState state = clusterService.state(); ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 7f59987617f2b..45b2d4712e39f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -694,7 +694,7 @@ private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScro } @Override - public DummyAbstractBulkByScrollRequest forSlice(TaskId slicingTask, SearchRequest slice) { + public DummyAbstractBulkByScrollRequest forSlice(TaskId slicingTask, SearchRequest slice, int totalSlices) { throw new UnsupportedOperationException(); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index 3ad48d803a437..5a378d4286f13 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -88,7 +88,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder createIndex(INDEX); // Total number of documents created for this test (~10 per primary shard per slice) - int numDocs = getNumShards(INDEX).numPrimaries * 10 * builder.request().getSlices(); + int numDocs = getNumShards(INDEX).numPrimaries * 10 * builder.request().getSlices().number(); ALLOWED_OPERATIONS.release(numDocs); indexRandom(true, false, true, IntStream.range(0, numDocs) @@ -104,8 +104,8 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder /* Allow a random number of the documents less the number of workers to be modified by the reindex action. That way at least one * worker is blocked. */ - int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs); - ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices()); + int numModifiedDocs = randomIntBetween(builder.request().getSlices().number() * 2, numDocs); + ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices().number()); // Now execute the reindex action... ActionFuture future = builder.execute(); @@ -115,7 +115,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder awaitBusy(() -> ALLOWED_OPERATIONS.hasQueuedThreads() && ALLOWED_OPERATIONS.availablePermits() == 0); // Status should show the task running - TaskInfo mainTask = findTaskToCancel(action, builder.request().getSlices()); + TaskInfo mainTask = findTaskToCancel(action, builder.request().getSlices().number()); BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus(); assertNull(status.getReasonCancelled()); @@ -132,7 +132,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder mainTask = client().admin().cluster().prepareGetTask(mainTask.getTaskId()).get().getTask().getTask(); status = (BulkByScrollTask.Status) mainTask.getStatus(); assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled()); - if (builder.request().getSlices() > 1) { + if (builder.request().getSlices().number() > 1) { boolean foundCancelled = false; ListTasksResponse sliceList = client().admin().cluster().prepareListTasks().setParentTaskId(mainTask.getTaskId()) .setDetailed(true).get(); @@ -147,11 +147,11 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder } // Unblock the last operations - ALLOWED_OPERATIONS.release(builder.request().getSlices()); + ALLOWED_OPERATIONS.release(builder.request().getSlices().number()); // Checks that no more operations are executed assertBusy(() -> { - if (builder.request().getSlices() == 1) { + if (builder.request().getSlices().number() == 1) { /* We can only be sure that we've drained all the permits if we only use a single worker. Otherwise some worker may have * exhausted all of its documents before we blocked. */ assertEquals(0, ALLOWED_OPERATIONS.availablePermits()); @@ -172,7 +172,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder assertThat(response.getBulkFailures(), emptyIterable()); assertThat(response.getSearchFailures(), emptyIterable()); - if (builder.request().getSlices() >= 1) { + if (builder.request().getSlices().number() >= 1) { // If we have more than one worker we might not have made all the modifications numModifiedDocs -= ALLOWED_OPERATIONS.availablePermits(); } @@ -232,7 +232,7 @@ public void testDeleteByQueryCancel() throws Exception { public void testReindexCancelWithWorkers() throws Exception { testCancel(ReindexAction.NAME, - reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(5), + reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(SlicesCount.of(5)), (response, total, modified) -> { assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); refresh("dest"); @@ -250,7 +250,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { "}"); assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); - testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5), + testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(SlicesCount.of(5)), (response, total, modified) -> { assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified); @@ -260,7 +260,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { } public void testDeleteByQueryCancelWithWorkers() throws Exception { - testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(5), + testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(SlicesCount.of(5)), (response, total, modified) -> { assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index aba7cd6935943..a0b64f35317f4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; @@ -237,13 +236,13 @@ public void testWorkers() throws Exception { assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); // Deletes the two docs that matches "foo:a" - assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(5).get(), + assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.of(5)).get(), matcher().deleted(2).slices(hasSize(5))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); // Delete remaining docs DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) - .setSlices(5); + .setSlices(SlicesCount.of(5)); assertThat(request.get(), matcher().deleted(5).slices(hasSize(5))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index 4f7753fca9abd..4d64e473c456d 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -100,7 +100,7 @@ public void testCopyManyWithSlices() throws Exception { assertHitCount(client().prepareSearch("source").setSize(0).get(), max); // Copy all the docs - ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(workers); + ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(SlicesCount.of(workers)); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(workers))); @@ -108,7 +108,7 @@ public void testCopyManyWithSlices() throws Exception { // Copy some of the docs int half = max / 2; - copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(workers); + copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(SlicesCount.of(workers)); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); copy.size(half); // The real "size" of the request. diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index 228bae4ed4a73..4af0dd3797a11 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -60,15 +60,15 @@ public void testDeleteByQuery() throws Exception { } public void testReindexWithWorkers() throws Exception { - testCase(reindex().source("test").destination("dest").setSlices(between(2, 10)), ReindexAction.NAME); + testCase(reindex().source("test").destination("dest").setSlices(SlicesCount.of(between(2, 10))), ReindexAction.NAME); } public void testUpdateByQueryWithWorkers() throws Exception { - testCase(updateByQuery().source("test").setSlices(between(2, 10)), UpdateByQueryAction.NAME); + testCase(updateByQuery().source("test").setSlices(SlicesCount.of(between(2, 10))), UpdateByQueryAction.NAME); } public void testDeleteByQueryWithWorkers() throws Exception { - testCase(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).setSlices(between(2, 10)), DeleteByQueryAction.NAME); + testCase(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).setSlices(SlicesCount.of(between(2, 10))), DeleteByQueryAction.NAME); } private void testCase(AbstractBulkByScrollRequestBuilder request, String actionName) throws Exception { @@ -77,7 +77,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a * we can't rely on all of them doing so, but */ List docs = new ArrayList<>(); - for (int i = 0; i < request.request().getSlices() * 10; i++) { + for (int i = 0; i < request.request().getSlices().number() * 10; i++) { // todo need to be aware for auto docs.add(client().prepareIndex("test", "test", Integer.toString(i)).setSource("foo", "bar")); } indexRandom(true, docs); @@ -87,15 +87,15 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a request.source().setSize(1); // Make sure we use multiple batches ActionFuture responseListener = request.execute(); - TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices()); + TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices().number()); TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId(); - if (request.request().getSlices() == 1) { + if (request.request().getSlices().number() == 1) { assertThat(taskGroupToRethrottle.getChildTasks(), empty()); } else { // There should be a sane number of child tasks running assertThat(taskGroupToRethrottle.getChildTasks(), - hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices())))); + hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices().number())))); // Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do) assertBusy(() -> { BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get() @@ -103,7 +103,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a long finishedSubTasks = parent.getSliceStatuses().stream().filter(Objects::nonNull).count(); ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get(); list.rethrowFailures("subtasks"); - assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) request.request().getSlices())); + assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) request.request().getSlices().number())); assertThat(list.getTasks().size(), greaterThan(0)); }); } @@ -115,7 +115,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a assertThat(rethrottleResponse.getTasks(), hasSize(1)); BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus(); // Now check the resulting requests per second. - if (request.request().getSlices() == 1) { + if (request.request().getSlices().number() == 1) { // If there is a single slice it should match perfectly assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL); } else { @@ -128,7 +128,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F; float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? - Float.POSITIVE_INFINITY : (newRequestsPerSecond / request.request().getSlices()) * 0.99F; + Float.POSITIVE_INFINITY : (newRequestsPerSecond / request.request().getSlices().number()) * 0.99F; boolean oneSliceRethrottled = false; float totalRequestsPerSecond = 0; for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) { @@ -164,7 +164,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a // Now the response should come back quickly because we've rethrottled the request BulkByScrollResponse response = responseListener.get(); assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", - response.getBatches(), greaterThanOrEqualTo(request.request().getSlices())); + response.getBatches(), greaterThanOrEqualTo(request.request().getSlices().number())); } private TaskGroup findTaskToRethrottle(String actionName, int sliceCount) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index 39806e475c7b8..31767034d897a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -71,14 +71,14 @@ public void testReindexRequest() throws IOException { assertRequestEquals(reindex, tripped); // Try slices with a version that doesn't support slices. That should fail. - reindex.setSlices(between(2, 1000)); + reindex.setSlices(SlicesCount.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, reindex, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new ReindexRequest(); - reindex.setSlices(1); + reindex.setSlices(SlicesCount.of(1)); roundTrip(Version.V_5_0_0_rc1, reindex, tripped); assertRequestEquals(Version.V_5_0_0_rc1, reindex, tripped); } @@ -95,14 +95,14 @@ public void testUpdateByQueryRequest() throws IOException { assertEquals(update.getPipeline(), tripped.getPipeline()); // Try slices with a version that doesn't support slices. That should fail. - update.setSlices(between(2, 1000)); + update.setSlices(SlicesCount.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, update, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new UpdateByQueryRequest(); - update.setSlices(1); + update.setSlices(SlicesCount.of(1)); roundTrip(Version.V_5_0_0_rc1, update, tripped); assertRequestEquals(update, tripped); assertEquals(update.getPipeline(), tripped.getPipeline()); @@ -116,14 +116,14 @@ public void testDeleteByQueryRequest() throws IOException { assertRequestEquals(delete, tripped); // Try slices with a version that doesn't support slices. That should fail. - delete.setSlices(between(2, 1000)); + delete.setSlices(SlicesCount.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, delete, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new DeleteByQueryRequest(); - delete.setSlices(1); + delete.setSlices(SlicesCount.of(1)); roundTrip(Version.V_5_0_0_rc1, delete, tripped); assertRequestEquals(delete, tripped); } @@ -139,7 +139,7 @@ private void randomRequest(AbstractBulkByScrollRequest request) { request.setTimeout(TimeValue.parseTimeValue(randomTimeValue(), null, "test")); request.setWaitForActiveShards(randomIntBetween(0, 10)); request.setRequestsPerSecond(between(0, Integer.MAX_VALUE)); - request.setSlices(between(1, Integer.MAX_VALUE)); + request.setSlices(SlicesCount.of(between(1, Integer.MAX_VALUE))); } private void randomRequest(AbstractBulkIndexByScrollRequest request) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java index 222aedd2e9eeb..26b2a6e502ee5 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java @@ -53,7 +53,8 @@ public class TransportRethrottleActionTests extends ESTestCase { @Before public void createTask() { slices = between(2, 50); - task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null, slices); + task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null); + task.setSlices(slices); } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index 663575a293391..c31f1220845a1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -73,18 +73,18 @@ public void testWorkers() throws Exception { assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); // Reindex all the docs - assertThat(updateByQuery().source("test").refresh(true).setSlices(5).get(), matcher().updated(4).slices(hasSize(5))); + assertThat(updateByQuery().source("test").refresh(true).setSlices(SlicesCount.of(5)).get(), matcher().updated(4).slices(hasSize(5))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now none of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(5).refresh(true).get(), + assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(SlicesCount.of(5)).refresh(true).get(), matcher().updated(0).slices(hasSize(5))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now half of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(5).get(), + assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.of(5)).get(), matcher().updated(2).slices(hasSize(5))); assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml index fe0c816ee14c0..6e911e3f1baae 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml @@ -45,6 +45,7 @@ - match: {throttled_millis: 0} - gte: { took: 0 } - is_false: task + - length: {slices: 5} - match: {slices.0.version_conflicts: 0} - match: {slices.0.throttled_millis: 0} - match: {slices.1.version_conflicts: 0} @@ -128,6 +129,7 @@ - match: {response.throttled_millis: 0} - gte: { response.took: 0 } - is_false: response.task + - length: {response.slices: 5} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} - match: {response.slices.1.version_conflicts: 0} @@ -142,6 +144,7 @@ - match: {task.status.deleted: 4} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 5} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} - match: {task.status.slices.1.version_conflicts: 0} @@ -252,6 +255,7 @@ - match: {response.throttled_millis: 0} - gte: { response.took: 0 } - is_false: response.task + - length: {response.slices: 2} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} - match: {response.slices.1.version_conflicts: 0} @@ -260,6 +264,7 @@ - match: {task.status.deleted: 6} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 2} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} - match: {task.status.slices.1.version_conflicts: 0} @@ -277,3 +282,72 @@ count: index: test - match: {count: 0} + +--- +"Multiple slices with auto slice": + - do: + indices.create: + index: test + body: + settings: + index: + number_of_shards: 3 + - do: + index: + index: test + type: foo + id: 1 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 2 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 3 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 4 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + delete_by_query: + index: test + slices: auto + body: + query: + match_all: {} + + - is_false: timed_out + - match: {deleted: 4} + - is_false: created + - is_false: updated + - match: {version_conflicts: 0} + - match: {failures: []} + - match: {noops: 0} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - is_false: task + - length: {slices: 3} + - match: {slices.0.version_conflicts: 0} + - match: {slices.0.throttled_millis: 0} + - match: {slices.1.version_conflicts: 0} + - match: {slices.1.throttled_millis: 0} + - match: {slices.2.version_conflicts: 0} + - match: {slices.2.throttled_millis: 0} + + - do: + indices.refresh: {} + - do: + count: + index: test + - match: {count: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml index 5c54adb5c088c..fb06018d7c086 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml @@ -43,6 +43,7 @@ - gte: { took: 0 } - is_false: task - is_false: deleted + - length: {slices: 5} - match: {slices.0.updated: 0} - match: {slices.0.version_conflicts: 0} - match: {slices.0.throttled_millis: 0} @@ -127,6 +128,7 @@ - gte: { response.took: 0 } - is_false: response.task - is_false: response.deleted + - length: {response.slices: 5} - match: {response.slices.0.updated: 0} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} @@ -147,6 +149,7 @@ - match: {task.status.updated: 0} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 5} - match: {task.status.slices.0.updated: 0} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} @@ -260,6 +263,7 @@ - gte: { response.took: 0 } - is_false: response.task - is_false: response.deleted + - length: {response.slices: 2} - match: {response.slices.0.updated: 0} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} @@ -271,6 +275,7 @@ - match: {task.status.updated: 0} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 2} - match: {task.status.slices.0.updated: 0} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} @@ -285,3 +290,67 @@ search: index: .tasks - match: { hits.total: 1 } + + +--- +"Multiple slices with auto slice": + - do: + indices.create: + index: source + body: + settings: + index: + number_of_shards: 3 + - do: + index: + index: source + type: foo + id: 1 + body: { "text": "test" } + - do: + index: + index: source + type: foo + id: 2 + body: { "text": "test" } + - do: + index: + index: source + type: foo + id: 3 + body: { "text": "test" } + - do: + index: + index: source + type: foo + id: 4 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + reindex: + slices: auto + body: + source: + index: source + dest: + index: dest + - match: {created: 4} + - match: {updated: 0} + - match: {version_conflicts: 0} + - match: {failures: []} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - is_false: task + - is_false: deleted + - length: {slices: 3} + - match: {slices.0.updated: 0} + - match: {slices.0.version_conflicts: 0} + - match: {slices.0.throttled_millis: 0} + - match: {slices.1.updated: 0} + - match: {slices.1.version_conflicts: 0} + - match: {slices.1.throttled_millis: 0} + - match: {slices.2.updated: 0} + - match: {slices.2.version_conflicts: 0} + - match: {slices.2.throttled_millis: 0} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml index f390d43f9f362..dbb79f8703725 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml @@ -44,6 +44,7 @@ - match: {throttled_millis: 0} - gte: { took: 0 } - is_false: task + - length: {slices: 5} - match: {slices.0.version_conflicts: 0} - match: {slices.0.throttled_millis: 0} - match: {slices.1.version_conflicts: 0} @@ -120,6 +121,7 @@ - match: {response.throttled_millis: 0} - gte: { response.took: 0 } - is_false: response.task + - length: {response.slices: 5} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} - match: {response.slices.1.version_conflicts: 0} @@ -134,6 +136,7 @@ - match: {task.status.updated: 4} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 5} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} - match: {task.status.slices.1.version_conflicts: 0} @@ -239,6 +242,7 @@ - match: {response.throttled_millis: 0} - gte: { response.took: 0 } - is_false: response.task + - length: {response.slices: 2} - match: {response.slices.0.version_conflicts: 0} - match: {response.slices.0.throttled_millis: 0} - match: {response.slices.1.version_conflicts: 0} @@ -247,6 +251,7 @@ - match: {task.status.updated: 6} - match: {task.status.version_conflicts: 0} - match: {task.status.throttled_millis: 0} + - length: {task.status.slices: 2} - match: {task.status.slices.0.version_conflicts: 0} - match: {task.status.slices.0.throttled_millis: 0} - match: {task.status.slices.1.version_conflicts: 0} @@ -259,3 +264,65 @@ search: index: .tasks - match: { hits.total: 1 } + + +--- +"Multiple slices with auto slice": + - do: + indices.create: + index: test + body: + settings: + index: + number_of_shards: 3 + - do: + index: + index: test + type: foo + id: 1 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 2 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 3 + body: { "text": "test" } + - do: + index: + index: test + type: foo + id: 4 + body: { "text": "test" } + - do: + indices.refresh: {} + + - do: + update_by_query: + index: test + slices: auto + body: + query: + match_all: {} + + - is_false: timed_out + - match: {updated: 4} + - is_false: created + - match: {version_conflicts: 0} + - match: {failures: []} + - match: {noops: 0} + - match: {throttled_millis: 0} + - gte: { took: 0 } + - is_false: task + - length: {slices: 3} + - match: {slices.0.version_conflicts: 0} + - match: {slices.0.throttled_millis: 0} + - match: {slices.1.version_conflicts: 0} + - match: {slices.1.throttled_millis: 0} + - match: {slices.2.version_conflicts: 0} + - match: {slices.2.throttled_millis: 0} From b7fd60db3464851413451f6f7bca982c9f4350b3 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 18 Jul 2017 11:56:30 -0700 Subject: [PATCH 03/21] wip #24547 Added similar unit tests for the three API types, they work so far --- .../reindex/DeleteByQueryBasicTests.java | 28 +++++++++++++++- .../index/reindex/ReindexBasicTests.java | 27 +++++++++++++++ .../reindex/UpdateByQueryBasicTests.java | 33 ++++++++++++++++++- 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index a0b64f35317f4..8d99809d1ac5c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -223,7 +223,7 @@ public void testDeleteByQueryOnReadOnlyIndex() throws Exception { assertHitCount(client().prepareSearch("test").setSize(0).get(), docs); } - public void testWorkers() throws Exception { + public void testSlices() throws Exception { indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"), client().prepareIndex("test", "test", "2").setSource("foo", "a"), @@ -247,6 +247,32 @@ public void testWorkers() throws Exception { assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } + public void testSlicesAuto() throws Exception { + indexRandom(true, + client().prepareIndex("test", "test", "1").setSource("foo", "a"), + client().prepareIndex("test", "test", "2").setSource("foo", "a"), + client().prepareIndex("test", "test", "3").setSource("foo", "b"), + client().prepareIndex("test", "test", "4").setSource("foo", "c"), + client().prepareIndex("test", "test", "5").setSource("foo", "d"), + client().prepareIndex("test", "test", "6").setSource("foo", "e"), + client().prepareIndex("test", "test", "7").setSource("foo", "f") + ); + assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); + + NumShards numShards = getNumShards("test"); + + // Deletes the two docs that matches "foo:a" + assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.AUTO).get(), + matcher().deleted(2).slices(hasSize(numShards.numPrimaries))); + assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); + + // Delete remaining docs + DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) + .setSlices(SlicesCount.AUTO); + assertThat(request.get(), matcher().deleted(5).slices(hasSize(numShards.numPrimaries))); + assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); + } + /** * Test delete by query support for filtering by type. This entire feature * can and should be removed when we drop support for types index with diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index 4d64e473c456d..5688252326063 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -116,4 +116,31 @@ public void testCopyManyWithSlices() throws Exception { assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(workers))); assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated()); } + + public void testCopyManyWithSlicesAuto() throws Exception { + int max = between(150, 500); + List docs = new ArrayList<>(); + for (int i = 0; i < max; i++) { + docs.add(client().prepareIndex("source", "test", Integer.toString(i)).setSource("foo", "a")); + } + + indexRandom(true, docs); + assertHitCount(client().prepareSearch("source").setSize(0).get(), max); + + NumShards numShards = getNumShards("source"); + + ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(SlicesCount.AUTO); + copy.source().setSize(5); + assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(numShards.numPrimaries))); + assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max); + + int half = max / 2; + copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(SlicesCount.AUTO); + copy.source().setSize(5); + copy.size(half); + BulkByScrollResponse response = copy.get(); + assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(numShards.numPrimaries))); + assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated()); + } + } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index c31f1220845a1..fb7ef4a026cef 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -63,7 +63,7 @@ public void testBasics() throws Exception { assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); } - public void testWorkers() throws Exception { + public void testSlices() throws Exception { indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"), client().prepareIndex("test", "test", "2").setSource("foo", "a"), client().prepareIndex("test", "test", "3").setSource("foo", "b"), @@ -91,4 +91,35 @@ public void testWorkers() throws Exception { assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); } + + public void testSlicesAuto() throws Exception { + indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"), + client().prepareIndex("test", "test", "2").setSource("foo", "a"), + client().prepareIndex("test", "test", "3").setSource("foo", "b"), + client().prepareIndex("test", "test", "4").setSource("foo", "c")); + assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 4); + assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion()); + assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); + + NumShards numShards = getNumShards("test"); + + // Reindex all the docs + assertThat(updateByQuery().source("test").refresh(true).setSlices(SlicesCount.AUTO).get(), matcher().updated(4).slices(hasSize(numShards.numPrimaries))); + assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); + assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); + + // Now none of them + assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(SlicesCount.AUTO).refresh(true).get(), + matcher().updated(0).slices(hasSize(numShards.numPrimaries))); + assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); + assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); + + // Now half of them + assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.AUTO).get(), + matcher().updated(2).slices(hasSize(numShards.numPrimaries))); + assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); + assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); + assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion()); + assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); + } } From 86392836bfcec9e34e81ebe650872de461418aaf Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 20 Jul 2017 10:18:44 -0700 Subject: [PATCH 04/21] wip #24547 Seemingly handles one-slice size correctly Broken when one slice and auto Next going to try refactoring the task types --- .../index/reindex/ParentBulkByScrollTask.java | 17 ++++++++ .../index/reindex/SlicesCount.java | 9 ++++ .../search/slice/SliceBuilder.java | 4 +- .../AbstractBulkByScrollRequestTestCase.java | 16 ++++--- .../reindex/ParentBulkByScrollTaskTests.java | 3 +- .../index/reindex/ReindexRequestTests.java | 4 +- .../search/slice/SliceBuilderTests.java | 4 +- .../BulkByScrollParallelizationHelper.java | 39 +++++++++++++++++ .../reindex/TransportDeleteByQueryAction.java | 33 ++++++--------- .../index/reindex/TransportReindexAction.java | 42 +++++++------------ .../reindex/TransportUpdateByQueryAction.java | 31 +++++--------- .../reindex/DeleteByQueryBasicTests.java | 3 ++ 12 files changed, 125 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java index 77db6269d0ff6..8594a38fa10fd 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java @@ -47,6 +47,9 @@ public class ParentBulkByScrollTask extends BulkByScrollTask { // todo either enforce slices being set before calling task methods or separate that out into another class // todo probably want to look at the tests for this class and make sure the slices set behavior is enforced + // todo this class may not be thread safe because the accesses to results and counter are not synchronized + // e.g. it's possible to read all task results finished and runningSubtasks != 0 at the same time + public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { super(id, type, action, description, parentTaskId); } @@ -118,6 +121,20 @@ public void onSliceResponse(ActionListener listener, int s recordSliceCompletionAndRespondIfAllDone(listener); } + + // todo this is not an atomic operation: you can read all the tasks results finished while counter is still not 0 + /* + * e.g. + * there are two child tasks + * + * thread A enters onSliceResponse + * thread A sets a result for child task 1 + * thread B enters onSliceResponse + * thread B enters a result for task 2 + * at this point all tasks are done + * thread B decrements counter + * thread A decrements counter and now thinks it's the last task even though it's not + */ /** * Record a failure from a slice and respond to the listener if the request is finished. */ diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java index d45e91c9b6fad..92a5e6323fdda 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java @@ -121,4 +121,13 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(count); } + + @Override + public String toString() { + if (isAuto()) { + return "auto"; + } else { + return Integer.toString(count); + } + } } diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index b09252889c276..c05b045db54bf 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -137,8 +137,8 @@ public int getId() { } private SliceBuilder setMax(int max) { - if (max <= 1) { - throw new IllegalArgumentException("max must be greater than 1"); + if (max <= 0) { // todo here + throw new IllegalArgumentException("max must be greater than 0"); } if (id != -1 && id >= max) { throw new IllegalArgumentException("max must be greater than id"); diff --git a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java index 8a255d376aff5..645b5be418dee 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java @@ -39,16 +39,22 @@ public void testForSlice() { randomFrom(ActiveShardCount.ALL, ActiveShardCount.NONE, ActiveShardCount.ONE, ActiveShardCount.DEFAULT)); original.setRetryBackoffInitialTime(parseTimeValue(randomPositiveTimeValue(), "retry_backoff_initial_time")); original.setMaxRetries(between(0, 1000)); - original.setSlices(between(2, 1000)); original.setRequestsPerSecond( randomBoolean() ? Float.POSITIVE_INFINITY : randomValueOtherThanMany(r -> r < 0, ESTestCase::randomFloat)); if (randomBoolean()) { original.setSize(between(0, Integer.MAX_VALUE)); } + int actualSlices = between(2, 1000); + if (randomBoolean()) { + original.setSlices(SlicesCount.of(actualSlices)); + } else { + original.setSlices(SlicesCount.AUTO); + } + TaskId slicingTask = new TaskId(randomAlphaOfLength(5), randomLong()); SearchRequest sliceRequest = new SearchRequest(); - R forSliced = original.forSlice(slicingTask, sliceRequest); + R forSliced = original.forSlice(slicingTask, sliceRequest, actualSlices); assertEquals(original.isAbortOnVersionConflict(), forSliced.isAbortOnVersionConflict()); assertEquals(original.isRefresh(), forSliced.isRefresh()); assertEquals(original.getTimeout(), forSliced.getTimeout()); @@ -56,11 +62,11 @@ public void testForSlice() { assertEquals(original.getRetryBackoffInitialTime(), forSliced.getRetryBackoffInitialTime()); assertEquals(original.getMaxRetries(), forSliced.getMaxRetries()); assertEquals("only the parent task should store results", false, forSliced.getShouldStoreResult()); - assertEquals("slice requests always have a single worker", 1, forSliced.getSlices()); - assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / original.getSlices(), + assertEquals("slice requests always have a single worker", SlicesCount.ONE, forSliced.getSlices()); + assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / actualSlices, forSliced.getRequestsPerSecond(), Float.MIN_NORMAL); assertEquals("size is split evenly between all workers", original.getSize() == AbstractBulkByScrollRequest.SIZE_ALL_MATCHES - ? AbstractBulkByScrollRequest.SIZE_ALL_MATCHES : original.getSize() / original.getSlices(), forSliced.getSize()); + ? AbstractBulkByScrollRequest.SIZE_ALL_MATCHES : original.getSize() / actualSlices, forSliced.getSize()); assertEquals(slicingTask, forSliced.getParentTask()); extraForSliceAssertions(original, forSliced); diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java index 6e2d44abed53a..fd6a992800193 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java @@ -40,7 +40,8 @@ public class ParentBulkByScrollTaskTests extends ESTestCase { @Before public void createTask() { slices = between(2, 50); - task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null, slices); + task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null); + task.setSlices(slices); } public void testBasicData() { diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index 32b01237375b2..fc67267f74e67 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -50,7 +50,7 @@ public void testReindexFromRemoteDoesNotSupportWorkers() { reindex.setRemoteInfo( new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); - reindex.setSlices(between(2, Integer.MAX_VALUE)); + reindex.setSlices(SlicesCount.of(between(2, Integer.MAX_VALUE))); ActionRequestValidationException e = reindex.validate(); assertEquals( "Validation Failed: 1: reindex from remote sources doesn't support workers > 1 but was [" + reindex.getSlices() + "];", @@ -60,7 +60,7 @@ public void testReindexFromRemoteDoesNotSupportWorkers() { public void testNoSliceWithWorkers() { ReindexRequest reindex = newRequest(); reindex.getSearchRequest().source().slice(new SliceBuilder(0, 4)); - reindex.setSlices(between(2, Integer.MAX_VALUE)); + reindex.setSlices(SlicesCount.of(between(2, Integer.MAX_VALUE))); ActionRequestValidationException e = reindex.validate(); assertEquals("Validation Failed: 1: can't specify both slice and workers;", e.getMessage()); } diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 009b9ab5d6535..a70a6f30be37a 100644 --- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -117,10 +117,10 @@ public void testInvalidArguments() throws Exception { assertEquals(e.getMessage(), "id must be greater than or equal to 0"); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, -1)); - assertEquals(e.getMessage(), "max must be greater than 1"); + assertEquals(e.getMessage(), "max must be greater than 0"); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 0)); - assertEquals(e.getMessage(), "max must be greater than 1"); + assertEquals(e.getMessage(), "max must be greater than 0"); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 5)); assertEquals(e.getMessage(), "max must be greater than id"); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 2208127f3b391..c4c00d4a130b3 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -21,20 +21,59 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + /** * Helps parallelize reindex requests using sliced scrolls. */ class BulkByScrollParallelizationHelper { + + public static final int AUTO_SLICE_CEILING = 20; + private BulkByScrollParallelizationHelper() {} + public static > void + computeSlicing(Client client, Request request, ActionListener listener, Consumer slicedBehavior) { + + SlicesCount slices = request.getSlices(); + if (slices.isAuto()) { + client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( + response -> slicedBehavior.accept(sliceBasedOnShards(response)), + exception -> listener.onFailure(exception) + )); + } else { + slicedBehavior.accept(request.getSlices().number()); + } + + } + + private static int sliceBasedOnShards(ClusterSearchShardsResponse response) { + Map countsByIndex = Arrays.stream(response.getGroups()).collect(Collectors.toMap( + group -> group.getShardId().getIndex(), + __ -> 1, + (sum, term) -> sum + term + )); + Set counts = new HashSet<>(countsByIndex.values()); + int leastShards = Collections.min(counts); + return Math.min(leastShards, AUTO_SLICE_CEILING); + } + public static > void startSlices(Client client, TaskManager taskManager, Action action, String localNodeId, ParentBulkByScrollTask task, Request request, diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 5ee5760ccc10f..9150318317425 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -51,29 +51,20 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { - - if (request.getSlices().isAuto()) { - client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( - r -> doDeleteByQuery(task, request, listener, r.getGroups().length), - e -> listener.onFailure(e) - )); - } else { - doDeleteByQuery(task, request, listener, request.getSlices().number()); - } - } - - private void doDeleteByQuery(Task task, DeleteByQueryRequest request, ActionListener listener, int slices) { - if (slices > 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, + BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { + logger.error("SLICES {} TASK {}", slices, task.getClass()); + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, clusterService.localNode().getId(), parentTask, request, slices, listener); - } else { - ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, + } else { + ClusterState state = clusterService.state(); + ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); + new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, listener).start(); - } + } + }); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 6c5a9ecfdecfa..f5b782e595862 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -35,7 +35,6 @@ import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse.Failure; import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; @@ -110,33 +109,22 @@ public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFi @Override protected void doExecute(Task task, ReindexRequest request, ActionListener listener) { - if (request.getSlices().isAuto()) { - // we need to look at shard details to decide on a slice count - client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( - r -> doReindex(task, request, listener, r.getGroups().length), - e -> listener.onFailure(e) - )); - } else { - // we already know an explicit slice count - doReindex(task, request, listener, request.getSlices().number()); - } - } - - private void doReindex(Task task, ReindexRequest request, ActionListener listener, int slices) { - if (slices > 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, clusterService.localNode().getId(), - parentTask, request, slices, listener); - } else { - checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); - ClusterState state = clusterService.state(); - validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), + BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, + clusterService.localNode().getId(), parentTask, request, slices, listener); + } else { + checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); + ClusterState state = clusterService.state(); + validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), indexNameExpressionResolver, autoCreateIndex, state); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, - listener).start(); - } + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); + new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, assigningClient, threadPool, request, scriptService, + state, listener).start(); + } + }); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 16d09ab1feffe..92dba8c0489a9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -64,28 +64,19 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { - if (request.getSlices().isAuto()) { - client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( - r -> doUpdateByQuery(task, request, listener, r.getGroups().length), - e -> listener.onFailure(e) - )); - } else { - doUpdateByQuery(task, request, listener, request.getSlices().number()); - } - } - - private void doUpdateByQuery(Task task, UpdateByQueryRequest request, ActionListener listener, int slices) { - if (slices > 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, + BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { + if (slices > 1) { + ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; + parentTask.setSlices(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, clusterService.localNode().getId(), parentTask, request, slices, listener); - } else { - ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, + } else { + ClusterState state = clusterService.state(); + ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); + new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, listener).start(); - } + } + }); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index 8d99809d1ac5c..03e46af73c8a1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -235,6 +235,8 @@ public void testSlices() throws Exception { ); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); + logger.error("SHARDSCOUNT {}", getNumShards("test").numPrimaries); + // Deletes the two docs that matches "foo:a" assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.of(5)).get(), matcher().deleted(2).slices(hasSize(5))); @@ -260,6 +262,7 @@ public void testSlicesAuto() throws Exception { assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); NumShards numShards = getNumShards("test"); + logger.error("SHARDSCOUNT {}", numShards.numPrimaries); // Deletes the two docs that matches "foo:a" assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.AUTO).get(), From d7491e082a2981520013c0ac3dfb87f7dc8974ad Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 20 Jul 2017 20:14:32 -0700 Subject: [PATCH 05/21] wip #24547 Changed the BulkByScroll task to use a strategy for the task behavior that used to all be built into the task Right now all the production code compiles except for a few things in xpack. Tests don't compile --- .../reindex/AbstractBulkByScrollRequest.java | 10 +- .../index/reindex/BulkByScrollTask.java | 71 +++-- .../reindex/ChildBulkByScrollWorker.java | 300 ++++++++++++++++++ .../index/reindex/ParentBulkByScrollTask.java | 28 +- .../reindex/ParentBulkByScrollWorker.java | 154 +++++++++ .../index/reindex/SuccessfullyProcessed.java | 2 +- .../reindex/WorkingBulkByScrollTask.java | 69 ++-- .../search/slice/SliceBuilder.java | 4 +- .../java/org/elasticsearch/tasks/Task.java | 2 +- .../AbstractAsyncBulkByScrollAction.java | 50 +-- .../reindex/AsyncDeleteByQueryAction.java | 2 +- .../BulkByScrollParallelizationHelper.java | 10 +- .../reindex/TransportDeleteByQueryAction.java | 20 +- .../index/reindex/TransportReindexAction.java | 38 ++- .../reindex/TransportRethrottleAction.java | 50 ++- .../reindex/TransportUpdateByQueryAction.java | 32 +- ...stractAsyncBulkByScrollActionTestCase.java | 6 +- 17 files changed, 726 insertions(+), 122 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java create mode 100644 core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 32cfe9f03bbf6..0ecae6da9ddae 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -382,14 +382,16 @@ protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId) { - if (slices.isAuto() || slices.number() > 1) { + /*if (slices.isAuto() || slices.number() > 1) { todo remove this return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId); } - /* Extract the slice from the search request so it'll be available in the status. This is potentially useful for users that manually + *//* Extract the slice from the search request so it'll be available in the status. This is potentially useful for users that manually * slice their search requests so they can keep track of it and **absolutely** useful for automatically sliced reindex requests so - * they can properly track the responses. */ + * they can properly track the responses. *//* Integer sliceId = searchRequest.source().slice() == null ? null : searchRequest.source().slice().getId(); - return new WorkingBulkByScrollTask(id, type, action, getDescription(), parentTaskId, sliceId, requestsPerSecond); + return new WorkingBulkByScrollTask(id, type, action, getDescription(), parentTaskId, sliceId, requestsPerSecond);*/ + + return new BulkByScrollTask(id, type, action, getDescription(), parentTaskId); } @Override diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 284fea7a38bfc..22d700a153779 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -39,36 +39,71 @@ import static java.lang.Math.min; import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableList; import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; /** * Task storing information about a currently running BulkByScroll request. */ -public abstract class BulkByScrollTask extends CancellableTask { +public class BulkByScrollTask extends CancellableTask { + + private ParentBulkByScrollWorker parentWorker; // todo give this a better name (maybe task strategy or behavior) + private ChildBulkByScrollWorker childWorker; + public BulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { super(id, type, action, description, parentTaskId); } - /** - * The number of sub-slices that are still running. {@link WorkingBulkByScrollTask} will always have 0 and - * {@link ParentBulkByScrollTask} will return the number of waiting tasks. Used to decide how to perform rethrottling. - */ - public abstract int runningSliceSubTasks(); + @Override + public BulkByScrollTask.Status getStatus() { + if (isParent()) { + return parentWorker.getStatus(); + } - /** - * Apply the {@code newRequestsPerSecond}. - */ - public abstract void rethrottle(float newRequestsPerSecond); + if (isChild()) { + return childWorker.getStatus(); + } - /* - * Overridden to force children to return compatible status. - */ - public abstract BulkByScrollTask.Status getStatus(); + throw new IllegalStateException("This task's worker is not set"); + } - /** - * Build the status for this task given a snapshot of the information of running slices. - */ - public abstract TaskInfo getInfoGivenSliceInfo(String localNodeId, List sliceInfo); + public boolean isParent() { + return parentWorker != null; + } + + public void setParent(int slices) { + if (isParent()) { + throw new IllegalStateException("Parent worker is already set"); + } + if (isChild()) { + throw new IllegalStateException("Worker is already set as child"); + } + + parentWorker = new ParentBulkByScrollWorker(this, slices); + } + + public ParentBulkByScrollWorker getParentWorker() { + return parentWorker; + } + + public boolean isChild() { + return childWorker != null; + } + + public void setChild(Integer sliceId, float requestsPerSecond) { + if (isChild()) { + throw new IllegalStateException("Child worker is already set"); + } + if (isParent()) { + throw new IllegalStateException("Worker is already set as child"); + } + + childWorker = new ChildBulkByScrollWorker(this, sliceId, requestsPerSecond); + } + + public ChildBulkByScrollWorker getChildWorker() { + return childWorker; + } @Override public boolean shouldCancelChildrenOnCancellation() { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java new file mode 100644 index 0000000000000..c6c43672516b9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -0,0 +1,300 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static java.lang.Math.max; +import static java.lang.Math.round; +import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; + +public class ChildBulkByScrollWorker implements SuccessfullyProcessed { + + private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName()); + + private final BulkByScrollTask task; + + /** + * The id of the slice that this worker is processing or {@code null} if this task isn't for a sliced request. + */ + private final Integer sliceId; + + /** + * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents + * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived. + */ + private final AtomicLong total = new AtomicLong(0); + private final AtomicLong updated = new AtomicLong(0); + private final AtomicLong created = new AtomicLong(0); + private final AtomicLong deleted = new AtomicLong(0); + private final AtomicLong noops = new AtomicLong(0); + private final AtomicInteger batch = new AtomicInteger(0); + private final AtomicLong versionConflicts = new AtomicLong(0); + private final AtomicLong bulkRetries = new AtomicLong(0); + private final AtomicLong searchRetries = new AtomicLong(0); + private final AtomicLong throttledNanos = new AtomicLong(); + + /** + * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX + * style variables but there isn't an AtomicFloat so we just use a volatile. + */ + private volatile float requestsPerSecond; + + /** + * Reference to any the last delayed prepareBulkRequest call. Used during rethrottling and canceling to reschedule the request. + */ + private final AtomicReference delayedPrepareBulkRequestReference = new AtomicReference<>(); + + public ChildBulkByScrollWorker(BulkByScrollTask task, Integer sliceId, float requestsPerSecond) { + this.task = task; + this.sliceId = sliceId; + setRequestsPerSecond(requestsPerSecond); + } + + public BulkByScrollTask.Status getStatus() { + return new BulkByScrollTask.Status(sliceId, total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), + noops.get(), bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), + task.getReasonCancelled(), throttledUntil()); + } + + public void setTotal(long totalHits) { + total.set(totalHits); + } + + public void countBatch() { + batch.incrementAndGet(); + } + + public void countNoop() { + noops.incrementAndGet(); + } + + @Override + public long getCreated() { + return created.get(); + } + + public void countCreated() { + created.incrementAndGet(); + } + + @Override + public long getUpdated() { + return updated.get(); + } + + public void countUpdated() { + updated.incrementAndGet(); + } + + @Override + public long getDeleted() { + return deleted.get(); + } + + public void countDeleted() { + deleted.incrementAndGet(); + } + + public void countVersionConflict() { + versionConflicts.incrementAndGet(); + } + + public void countBulkRetry() { + bulkRetries.incrementAndGet(); + } + + public void countSearchRetry() { + searchRetries.incrementAndGet(); + } + + float getRequestsPerSecond() { + return requestsPerSecond; + } + + TimeValue throttledUntil() { + DelayedPrepareBulkRequest delayed = delayedPrepareBulkRequestReference.get(); + if (delayed == null) { + return timeValueNanos(0); + } + if (delayed.future == null) { + return timeValueNanos(0); + } + return timeValueNanos(max(0, delayed.future.getDelay(TimeUnit.NANOSECONDS))); + } + + /** + * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be + * rescheduled over and over again. + */ + public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize, + AbstractRunnable prepareBulkRequestRunnable) { + // Synchronize so we are less likely to schedule the same request twice. + synchronized (delayedPrepareBulkRequestReference) { + TimeValue delay = throttleWaitTime(lastBatchStartTime, timeValueNanos(System.nanoTime()), lastBatchSize); + logger.debug("[{}]: preparing bulk request for [{}]", task.getId(), delay); + delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(), + delay, new RunOnce(prepareBulkRequestRunnable))); + } + } + + public TimeValue throttleWaitTime(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) { + long earliestNextBatchStartTime = now.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize); + return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())); + } + + /** + * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing. + */ + float perfectlyThrottledBatchTime(int lastBatchSize) { + if (requestsPerSecond == Float.POSITIVE_INFINITY) { + return 0; + } + // requests + // ------------------- == seconds + // request per seconds + float targetBatchTimeInSeconds = lastBatchSize / requestsPerSecond; + // nanoseconds per seconds * seconds == nanoseconds + return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds; + } + + private void setRequestsPerSecond(float requestsPerSecond) { + if (requestsPerSecond <= 0) { + throw new IllegalArgumentException("requests per second must be more than 0 but was [" + requestsPerSecond + "]"); + } + this.requestsPerSecond = requestsPerSecond; + } + + public void rethrottle(float newRequestsPerSecond) { + synchronized (delayedPrepareBulkRequestReference) { + logger.debug("[{}]: rethrottling to [{}] requests per second", task.getId(), newRequestsPerSecond); + setRequestsPerSecond(newRequestsPerSecond); + + DelayedPrepareBulkRequest delayedPrepareBulkRequest = this.delayedPrepareBulkRequestReference.get(); + if (delayedPrepareBulkRequest == null) { + // No request has been queued so nothing to reschedule. + logger.debug("[{}]: skipping rescheduling because there is no scheduled task", task.getId()); + return; + } + + this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond)); + } + } + + class DelayedPrepareBulkRequest { + private final ThreadPool threadPool; + private final AbstractRunnable command; + private final float requestsPerSecond; + private final ScheduledFuture future; + + DelayedPrepareBulkRequest(ThreadPool threadPool, float requestsPerSecond, TimeValue delay, AbstractRunnable command) { + this.threadPool = threadPool; + this.requestsPerSecond = requestsPerSecond; + this.command = command; + this.future = threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + throttledNanos.addAndGet(delay.nanos()); + command.run(); + } + + @Override + public void onFailure(Exception e) { + command.onFailure(e); + } + }); + } + + DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) { + if (newRequestsPerSecond < requestsPerSecond) { + /* The user is attempting to slow the request down. We'll let the + * change in throttle take effect the next time we delay + * prepareBulkRequest. We can't just reschedule the request further + * out in the future because the bulk context might time out. */ + logger.debug("[{}]: skipping rescheduling because the new throttle [{}] is slower than the old one [{}]", task.getId(), + newRequestsPerSecond, requestsPerSecond); + return this; + } + + long remainingDelay = future.getDelay(TimeUnit.NANOSECONDS); + // Actually reschedule the task + if (false == FutureUtils.cancel(future)) { + // Couldn't cancel, probably because the task has finished or been scheduled. Either way we have nothing to do here. + logger.debug("[{}]: skipping rescheduling because we couldn't cancel the task", task.getId()); + return this; + } + + /* Strangely enough getting here doesn't mean that you actually + * cancelled the request, just that you probably did. If you stress + * test it you'll find that requests sneak through. So each request + * is given a runOnce boolean to prevent that. */ + TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond); + logger.debug("[{}]: rescheduling for [{}] in the future", task.getId(), newDelay); + return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command); + } + + /** + * Scale back remaining delay to fit the new delay. + */ + TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) { + if (remainingDelay < 0) { + return timeValueNanos(0); + } + return timeValueNanos(round(remainingDelay * requestsPerSecond / newRequestsPerSecond)); + } + } + + /** + * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times. + * Without it the command would be run multiple times. + */ + private static class RunOnce extends AbstractRunnable { + private final AtomicBoolean hasRun = new AtomicBoolean(false); + private final AbstractRunnable delegate; + + RunOnce(AbstractRunnable delegate) { + this.delegate = delegate; + } + + @Override + protected void doRun() throws Exception { + if (hasRun.compareAndSet(false, true)) { + delegate.run(); + } + } + + @Override + public void onFailure(Exception e) { + delegate.onFailure(e); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java index 8594a38fa10fd..af69419d8a861 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java @@ -33,13 +33,15 @@ import static java.util.Collections.unmodifiableList; /** - * Task for parent bulk by scroll requests that have sub-workers. + * Task for parent bulk by scroll requests that have sub-workers. todo remove */ public class ParentBulkByScrollTask extends BulkByScrollTask { + public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { + super(id, type, action, description, parentTaskId); + } /** * Holds the responses as they come back. This uses {@link Tuple} as an "Either" style holder where only the response or the exception * is set. - */ private AtomicArray results; private AtomicInteger counter; @@ -89,8 +91,8 @@ public int runningSliceSubTasks() { @Override public TaskInfo getInfoGivenSliceInfo(String localNodeId, List sliceInfo) { - /* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the - * finished status because we don't expect them to change after the task is finished. */ + *//* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the + * finished status because we don't expect them to change after the task is finished. *//* List sliceStatuses = Arrays.asList(new StatusOrException[results.length()]); for (TaskInfo t : sliceInfo) { Status status = (Status) t.getStatus(); @@ -111,19 +113,19 @@ private void addResultsToList(List sliceStatuses) { } } - /** + *//** * Record a response from a slice and respond to the listener if the request is finished. - */ + *//* public void onSliceResponse(ActionListener listener, int sliceId, BulkByScrollResponse response) { results.setOnce(sliceId, new Result(sliceId, response)); - /* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we - * were fairly sure they had a while left to go. */ + *//* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we + * were fairly sure they had a while left to go. *//* recordSliceCompletionAndRespondIfAllDone(listener); } // todo this is not an atomic operation: you can read all the tasks results finished while counter is still not 0 - /* + *//* * e.g. * there are two child tasks * @@ -134,10 +136,10 @@ public void onSliceResponse(ActionListener listener, int s * at this point all tasks are done * thread B decrements counter * thread A decrements counter and now thinks it's the last task even though it's not - */ - /** + *//* + *//** * Record a failure from a slice and respond to the listener if the request is finished. - */ + *//* public void onSliceFailure(ActionListener listener, int sliceId, Exception e) { results.setOnce(sliceId, new Result(sliceId, e)); recordSliceCompletionAndRespondIfAllDone(listener); @@ -186,5 +188,5 @@ private Result(int sliceId, Exception failure) { this.failure = failure; response = null; } - } + }*/ } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java new file mode 100644 index 0000000000000..447b0dd9e7383 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java @@ -0,0 +1,154 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.tasks.TaskInfo; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static java.util.Collections.unmodifiableList; + +public class ParentBulkByScrollWorker { + + private final BulkByScrollTask task; + + private final int slices; + /** + * Holds the responses of slice workers as they come in + */ + private final AtomicArray results; + /** + * How many subtasks are still running TODO rename this + */ + private final AtomicInteger counter; + + public ParentBulkByScrollWorker(BulkByScrollTask task, int slices) { + this.task = task; + this.slices = slices; + results = new AtomicArray<>(slices); + counter = new AtomicInteger(slices); + } + + public int getSlices() { + return slices; + } + + public BulkByScrollTask.Status getStatus() { // moved to parent worker + // We only have access to the statuses of requests that have finished so we return them + List statuses = Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()]); + addResultsToList(statuses); + return new BulkByScrollTask.Status(unmodifiableList(statuses), task.getReasonCancelled()); + } + + public int runningSliceSubTasks() { + return counter.get(); + } + + public TaskInfo getStatusGivenSlicesTaskInfo(String localNodeId, List sliceInfo) { + /* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the + * finished status because we don't expect them to change after the task is finished. */ + List sliceStatuses = Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()]); + for (TaskInfo t : sliceInfo) { + BulkByScrollTask.Status status = (BulkByScrollTask.Status) t.getStatus(); + sliceStatuses.set(status.getSliceId(), new BulkByScrollTask.StatusOrException(status)); + } + addResultsToList(sliceStatuses); + BulkByScrollTask.Status status = new BulkByScrollTask.Status(sliceStatuses, task.getReasonCancelled()); + return task.taskInfo(localNodeId, task.getDescription(), status); + } + + private void addResultsToList(List sliceStatuses) { + for (Result t : results.asList()) { + if (t.response != null) { + sliceStatuses.set(t.sliceId, new BulkByScrollTask.StatusOrException(t.response.getStatus())); + } else { + sliceStatuses.set(t.sliceId, new BulkByScrollTask.StatusOrException(t.failure)); + } + } + } + + /** + * Record a response from a slice and respond to the listener if the request is finished. + */ + public void onSliceResponse(ActionListener listener, int sliceId, BulkByScrollResponse response) { + results.setOnce(sliceId, new Result(sliceId, response)); + /* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we + * were fairly sure they had a while left to go. */ + recordSliceCompletionAndRespondIfAllDone(listener); + } + + /** + * Record a failure from a slice and respond to the listener if the request is finished. + */ + public void onSliceFailure(ActionListener listener, int sliceId, Exception e) { + results.setOnce(sliceId, new Result(sliceId, e)); + recordSliceCompletionAndRespondIfAllDone(listener); + // TODO cancel when a slice fails? + } + + private void recordSliceCompletionAndRespondIfAllDone(ActionListener listener) { + if (counter.decrementAndGet() != 0) { + return; + } + List responses = new ArrayList<>(results.length()); + Exception exception = null; + for (Result t : results.asList()) { + if (t.response == null) { + assert t.failure != null : "exception shouldn't be null if value is null"; + if (exception == null) { + exception = t.failure; + } else { + exception.addSuppressed(t.failure); + } + } else { + assert t.failure == null : "exception should be null if response is not null"; + responses.add(t.response); + } + } + if (exception == null) { + listener.onResponse(new BulkByScrollResponse(responses, task.getReasonCancelled())); + } else { + listener.onFailure(exception); + } + } + + private static final class Result { + final BulkByScrollResponse response; + final int sliceId; + final Exception failure; + + private Result(int sliceId, BulkByScrollResponse response) { + this.sliceId = sliceId; + this.response = response; + failure = null; + } + + private Result(int sliceId, Exception failure) { + this.sliceId = sliceId; + this.failure = failure; + response = null; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java index 6547984900e4b..34c63c50c65a1 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.reindex; /** - * Implemented by {@link BulkByScrollTask} and {@link BulkByScrollTask.Status} to consistently implement + * Implemented by {@link ChildBulkByScrollWorker} and {@link BulkByScrollTask.Status} to consistently implement * {@link #getSuccessfullyProcessed()}. */ public interface SuccessfullyProcessed { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java index 4e11b3c9595e0..35c86170a646b 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java @@ -41,19 +41,37 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; /** - * {@link BulkByScrollTask} subclass for tasks that actually perform the work. Compare to {@link ParentBulkByScrollTask}. + * todo remove. */ public class WorkingBulkByScrollTask extends BulkByScrollTask implements SuccessfullyProcessed { - private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName()); + public WorkingBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { + super(id, type, action, description, parentTaskId); + } + + @Override + public long getUpdated() { + return 0; + } + + @Override + public long getCreated() { + return 0; + } - /** + @Override + public long getDeleted() { + return 0; + } + /*private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName()); + + *//** * The id of the slice that this task is processing or {@code null} if this task isn't for a sliced request. - */ + *//* private final Integer sliceId; - /** + *//** * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived. - */ + *//* private final AtomicLong total = new AtomicLong(0); private final AtomicLong updated = new AtomicLong(0); private final AtomicLong created = new AtomicLong(0); @@ -64,14 +82,14 @@ public class WorkingBulkByScrollTask extends BulkByScrollTask implements Success private final AtomicLong bulkRetries = new AtomicLong(0); private final AtomicLong searchRetries = new AtomicLong(0); private final AtomicLong throttledNanos = new AtomicLong(); - /** + *//** * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX * style variables but there isn't an AtomicFloat so we just use a volatile. - */ + *//* private volatile float requestsPerSecond; - /** + *//** * Reference to any the last delayed prepareBulkRequest call. Used during rethrottling and canceling to reschedule the request. - */ + *//* private final AtomicReference delayedPrepareBulkRequestReference = new AtomicReference<>(); public WorkingBulkByScrollTask(long id, String type, String action, String description, TaskId parentTask, Integer sliceId, @@ -90,8 +108,8 @@ public Status getStatus() { @Override protected void onCancelled() { - /* Drop the throttle to 0, immediately rescheduling any throttled - * operation so it will wake up and cancel itself. */ + Drop the throttle to 0, immediately rescheduling any throttled + * operation so it will wake up and cancel itself. rethrottle(Float.POSITIVE_INFINITY); } @@ -171,10 +189,10 @@ float getRequestsPerSecond() { return requestsPerSecond; } - /** + *//** * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be * rescheduled over and over again. - */ + *//* public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize, AbstractRunnable prepareBulkRequestRunnable) { // Synchronize so we are less likely to schedule the same request twice. @@ -191,9 +209,9 @@ public TimeValue throttleWaitTime(TimeValue lastBatchStartTime, TimeValue now, i return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())); } - /** + *//** * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing. - */ + *//* float perfectlyThrottledBatchTime(int lastBatchSize) { if (requestsPerSecond == Float.POSITIVE_INFINITY) { return 0; @@ -213,6 +231,7 @@ private void setRequestsPerSecond(float requestsPerSecond) { this.requestsPerSecond = requestsPerSecond; } + @Override public void rethrottle(float newRequestsPerSecond) { synchronized (delayedPrepareBulkRequestReference) { @@ -256,10 +275,10 @@ public void onFailure(Exception e) { DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) { if (newRequestsPerSecond < requestsPerSecond) { - /* The user is attempting to slow the request down. We'll let the + *//* The user is attempting to slow the request down. We'll let the * change in throttle take effect the next time we delay * prepareBulkRequest. We can't just reschedule the request further - * out in the future because the bulk context might time out. */ + * out in the future because the bulk context might time out. *//* logger.debug("[{}]: skipping rescheduling because the new throttle [{}] is slower than the old one [{}]", getId(), newRequestsPerSecond, requestsPerSecond); return this; @@ -273,18 +292,18 @@ DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) { return this; } - /* Strangely enough getting here doesn't mean that you actually + *//* Strangely enough getting here doesn't mean that you actually * cancelled the request, just that you probably did. If you stress * test it you'll find that requests sneak through. So each request - * is given a runOnce boolean to prevent that. */ + * is given a runOnce boolean to prevent that. *//* TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond); logger.debug("[{}]: rescheduling for [{}] in the future", getId(), newDelay); return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command); } - /** + *//** * Scale back remaining delay to fit the new delay. - */ + *//* TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) { if (remainingDelay < 0) { return timeValueNanos(0); @@ -293,10 +312,10 @@ TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) { } } - /** + *//** * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times. * Without it the command would be run multiple times. - */ + *//* private static class RunOnce extends AbstractRunnable { private final AtomicBoolean hasRun = new AtomicBoolean(false); private final AbstractRunnable delegate; @@ -316,5 +335,5 @@ protected void doRun() throws Exception { public void onFailure(Exception e) { delegate.onFailure(e); } - } + }*/ } diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index c05b045db54bf..b09252889c276 100644 --- a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -137,8 +137,8 @@ public int getId() { } private SliceBuilder setMax(int max) { - if (max <= 0) { // todo here - throw new IllegalArgumentException("max must be greater than 0"); + if (max <= 1) { + throw new IllegalArgumentException("max must be greater than 1"); } if (id != -1 && id >= max) { throw new IllegalArgumentException("max must be greater than id"); diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java index bc2e8418141dc..0c0228d5e0167 100644 --- a/core/src/main/java/org/elasticsearch/tasks/Task.java +++ b/core/src/main/java/org/elasticsearch/tasks/Task.java @@ -89,7 +89,7 @@ public final TaskInfo taskInfo(String localNodeId, boolean detailed) { /** * Build a proper {@link TaskInfo} for this task. */ - protected final TaskInfo taskInfo(String localNodeId, String description, Status status) { + public final TaskInfo taskInfo(String localNodeId, String description, Status status) { return new TaskInfo(new TaskId(localNodeId, getId()), getType(), getAction(), description, status, startTime, System.nanoTime() - startTimeNanos, this instanceof CancellableTask, parentTask); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 91673fd0a41b1..2da9517941958 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -87,7 +87,8 @@ */ public abstract class AbstractAsyncBulkByScrollAction> { protected final Logger logger; - protected final WorkingBulkByScrollTask task; + protected final BulkByScrollTask task; + protected final ChildBulkByScrollWorker worker; protected final ThreadPool threadPool; protected final ScriptService scriptService; protected final ClusterState clusterState; @@ -114,16 +115,22 @@ public abstract class AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> scriptApplier; - public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { this(task, logger, client, threadPool, mainRequest, scriptService, clusterState, listener, client.settings()); } - public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState, ActionListener listener, Settings settings) { + this.task = task; + if (!task.isChild()) { + throw new IllegalArgumentException("Given task [" + task.getId() + "] must have a child worker"); + } + this.worker = task.getChildWorker(); + this.logger = logger; this.client = client; this.settings = settings; @@ -133,7 +140,7 @@ public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logg this.mainRequest = mainRequest; this.listener = listener; BackoffPolicy backoffPolicy = buildBackoffPolicy(); - bulkRetry = new Retry(EsRejectedExecutionException.class, BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry), threadPool); + bulkRetry = new Retry(EsRejectedExecutionException.class, BackoffPolicy.wrap(backoffPolicy, worker::countBulkRetry), threadPool); scrollSource = buildScrollableResultSource(backoffPolicy); scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null"); /* @@ -217,7 +224,7 @@ private BulkRequest buildBulk(Iterable docs) } protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) { - return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client, + return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, client, mainRequest.getSearchRequest()); } @@ -272,7 +279,7 @@ void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, Scrollabl if (mainRequest.getSize() > 0) { total = min(total, mainRequest.getSize()); } - task.setTotal(total); + worker.setTotal(total); AbstractRunnable prepareBulkRequestRunnable = new AbstractRunnable() { @Override protected void doRun() throws Exception { @@ -289,7 +296,7 @@ public void onFailure(Exception e) { } }; prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable); - task.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable); + worker.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable); } /** @@ -308,11 +315,11 @@ void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Respon refreshAndFinish(emptyList(), emptyList(), false); return; } - task.countBatch(); + worker.countBatch(); List hits = response.getHits(); if (mainRequest.getSize() != SIZE_ALL_MATCHES) { // Truncate the hits if we have more than the request size - long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed()); + long remaining = max(0, mainRequest.getSize() - worker.getSuccessfullyProcessed()); if (remaining < hits.size()) { hits = hits.subList(0, (int) remaining); } @@ -372,16 +379,16 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) { case CREATE: case INDEX: if (item.getResponse().getResult() == DocWriteResponse.Result.CREATED) { - task.countCreated(); + worker.countCreated(); } else { - task.countUpdated(); + worker.countUpdated(); } break; case UPDATE: - task.countUpdated(); + worker.countUpdated(); break; case DELETE: - task.countDeleted(); + worker.countDeleted(); break; } // Track the indexes we've seen so we can refresh them if requested @@ -401,7 +408,7 @@ void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) { return; } - if (mainRequest.getSize() != SIZE_ALL_MATCHES && task.getSuccessfullyProcessed() >= mainRequest.getSize()) { + if (mainRequest.getSize() != SIZE_ALL_MATCHES && worker.getSuccessfullyProcessed() >= mainRequest.getSize()) { // We've processed all the requested docs. refreshAndFinish(emptyList(), emptyList(), false); return; @@ -425,7 +432,7 @@ void startNextScroll(TimeValue lastBatchStartTime, TimeValue now, int lastBatchS finishHim(null); return; } - TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, now, lastBatchSize); + TimeValue extraKeepAlive = worker.throttleWaitTime(lastBatchStartTime, now, lastBatchSize); scrollSource.startNextScroll(extraKeepAlive, response -> { onScrollResponse(lastBatchStartTime, lastBatchSize, response); }); @@ -433,7 +440,7 @@ void startNextScroll(TimeValue lastBatchStartTime, TimeValue now, int lastBatchS private void recordFailure(Failure failure, List failures) { if (failure.getStatus() == CONFLICT) { - task.countVersionConflict(); + worker.countVersionConflict(); if (false == mainRequest.isAbortOnVersionConflict()) { return; } @@ -759,9 +766,9 @@ public static RequestWrapper wrap(DeleteRequest request) { /** * Apply a {@link Script} to a {@link RequestWrapper} */ - public abstract class ScriptApplier implements BiFunction, ScrollableHitSource.Hit, RequestWrapper> { + public static abstract class ScriptApplier implements BiFunction, ScrollableHitSource.Hit, RequestWrapper> { - private final WorkingBulkByScrollTask task; + private final ChildBulkByScrollWorker taskWorker; private final ScriptService scriptService; private final Script script; private final Map params; @@ -769,9 +776,8 @@ public abstract class ScriptApplier implements BiFunction, Scr private ExecutableScript executable; private Map context; - public ScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script, - Map params) { - this.task = task; + public ScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, Map params) { + this.taskWorker = taskWorker; this.scriptService = scriptService; this.script = script; this.params = params; @@ -864,7 +870,7 @@ public RequestWrapper apply(RequestWrapper request, ScrollableHitSource.Hi protected RequestWrapper scriptChangedOpType(RequestWrapper request, OpType oldOpType, OpType newOpType) { switch (newOpType) { case NOOP: - task.countNoop(); + taskWorker.countNoop(); return null; case DELETE: RequestWrapper delete = wrap(new DeleteRequest(request.getIndex(), request.getType(), request.getId())); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java index 2608f5715ba3e..8dd30a9fa9d65 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AsyncDeleteByQueryAction.java @@ -31,7 +31,7 @@ * Implementation of delete-by-query using scrolling and bulk. */ public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction { - public AsyncDeleteByQueryAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + public AsyncDeleteByQueryAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { super(task, logger, client, threadPool, request, scriptService, clusterState, listener); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index c4c00d4a130b3..93f0e26655980 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -76,16 +76,18 @@ private static int sliceBasedOnShards(ClusterSearchShardsResponse response) { public static > void startSlices(Client client, TaskManager taskManager, Action action, - String localNodeId, ParentBulkByScrollTask task, Request request, - int totalSlices, + String localNodeId, + BulkByScrollTask task, + Request request, ActionListener listener) { + int totalSlices = task.getParentWorker().getSlices(); TaskId parentTaskId = new TaskId(localNodeId, task.getId()); for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, totalSlices)) { // TODO move the request to the correct node. maybe here or somehow do it as part of startup for reindex in general.... Request requestForSlice = request.forSlice(parentTaskId, slice, totalSlices); ActionListener sliceListener = ActionListener.wrap( - r -> task.onSliceResponse(listener, slice.source().slice().getId(), r), - e -> task.onSliceFailure(listener, slice.source().slice().getId(), e)); + r -> task.getParentWorker().onSliceResponse(listener, slice.source().slice().getId(), r), + e -> task.getParentWorker().onSliceFailure(listener, slice.source().slice().getId(), e)); client.execute(action, requestForSlice, sliceListener); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 9150318317425..1c8a5682113fb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -53,7 +53,25 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { logger.error("SLICES {} TASK {}", slices, task.getClass()); + + BulkByScrollTask bulkTask = (BulkByScrollTask) task; + if (slices > 1) { + bulkTask.setParent(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, + clusterService.localNode().getId(), bulkTask, request, listener); + } else { + Integer sliceId = request.getSearchRequest().source().slice() == null + ? null + : request.getSearchRequest().source().slice().getId(); + bulkTask.setChild(sliceId, request.getRequestsPerSecond()); + ClusterState state = clusterService.state(); + ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), bulkTask); + new AsyncDeleteByQueryAction(bulkTask, logger, client, threadPool, request, scriptService, state, listener).start(); + } + + + /*if (slices > 1) { ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, @@ -63,7 +81,7 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { + + BulkByScrollTask bulkTask = (BulkByScrollTask) task; + if (slices > 1) { + bulkTask.setParent(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, + clusterService.localNode().getId(), bulkTask, request, listener); + } else { + checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); + ClusterState state = clusterService.state(); + validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), + indexNameExpressionResolver, autoCreateIndex, state); + + Integer sliceId = request.getSearchRequest().source().slice() == null + ? null + : request.getSearchRequest().source().slice().getId(); + bulkTask.setChild(sliceId, request.getRequestsPerSecond()); + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkTask); + new AsyncIndexBySearchAction(bulkTask, logger, assigningClient, threadPool, request, scriptService, state, listener).start(); + } + +/* if (slices > 1) { ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, @@ -123,7 +146,7 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener createdThreads = emptyList(); - AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { this(task, logger, client, threadPool, request, scriptService, clusterState, listener, client.settings()); } - AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener, Settings settings) { super(task, logger, client, threadPool, request, scriptService, clusterState, listener, settings); @@ -275,7 +298,7 @@ protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffP RemoteInfo remoteInfo = mainRequest.getRemoteInfo(); createdThreads = synchronizedList(new ArrayList<>()); RestClient restClient = buildRestClient(remoteInfo, task.getId(), createdThreads); - return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, restClient, + return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, restClient, remoteInfo.getQuery(), mainRequest.getSearchRequest()); } return super.buildScrollableResultSource(backoffPolicy); @@ -297,7 +320,7 @@ protected void finishHim(Exception failure, List indexingFailures, List public BiFunction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new ReindexScriptApplier(task, scriptService, script, script.getParams()); + return new ReindexScriptApplier(worker, scriptService, script, script.getParams()); } return super.buildScriptApplier(); } @@ -389,9 +412,8 @@ protected void copyRouting(RequestWrapper request, String routing) { class ReindexScriptApplier extends ScriptApplier { - ReindexScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script, - Map params) { - super(task, scriptService, script, params); + public ReindexScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, Map params) { + super(taskWorker, scriptService, script, params); } /* diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index bcfb281347496..4c5282c4312e9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -59,21 +59,45 @@ protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, A static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { - int runningSubTasks = task.runningSliceSubTasks(); - if (runningSubTasks == 0) { - logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond); - task.rethrottle(newRequestsPerSecond); - listener.onResponse(task.taskInfo(localNodeId, true)); + + if (task.isChild()) { + rethrottleChildTask(logger, localNodeId, task, newRequestsPerSecond, listener); + return; + } + + if (task.isParent()) { + rethrottleParentTask(logger, localNodeId, client, task, newRequestsPerSecond, listener); return; } - RethrottleRequest subRequest = new RethrottleRequest(); - subRequest.setRequestsPerSecond(newRequestsPerSecond / runningSubTasks); - subRequest.setParentTaskId(new TaskId(localNodeId, task.getId())); - logger.debug("rethrottling children of task [{}] to [{}] requests per second", task.getId(), subRequest.getRequestsPerSecond()); - client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap(r -> { - r.rethrowFailures("Rethrottle"); - listener.onResponse(task.getInfoGivenSliceInfo(localNodeId, r.getTasks())); - }, listener::onFailure)); + + throw new IllegalArgumentException("task [" + task.getId() + "] must be set as a child or parent"); + } + + private static void rethrottleParentTask(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { + final ParentBulkByScrollWorker parentWorker = task.getParentWorker(); + final int runningSubtasks = parentWorker.runningSliceSubTasks(); + + if (runningSubtasks > 0) { + RethrottleRequest subRequest = new RethrottleRequest(); + subRequest.setRequestsPerSecond(newRequestsPerSecond / runningSubtasks); + subRequest.setParentTask(new TaskId(localNodeId, task.getId())); + logger.debug("rethrottling children of task [{}] to [{}] requests per second", task.getId(), + subRequest.getRequestsPerSecond()); + client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap( + r -> { + r.rethrowFailures("Rethrottle"); + listener.onResponse(parentWorker.getStatusGivenSlicesTaskInfo(localNodeId, r.getTasks())); + }, + listener::onFailure)); + } else { + logger.debug("children of task [{}] are already finished, nothing to rethrottle", task.getId()); + } + } + + private static void rethrottleChildTask(Logger logger, String localNodeId, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { + logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond); + task.getChildWorker().rethrottle(newRequestsPerSecond); + listener.onResponse(task.taskInfo(localNodeId, true)); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 92dba8c0489a9..77d3c5a6c1796 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -65,7 +65,25 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { + + BulkByScrollTask bulkTask = (BulkByScrollTask) task; + if (slices > 1) { + bulkTask.setParent(slices); + BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, + clusterService.localNode().getId(), bulkTask, request, listener); + } else { + Integer sliceId = request.getSearchRequest().source().slice() == null + ? null + : request.getSearchRequest().source().slice().getId(); + bulkTask.setChild(sliceId, request.getRequestsPerSecond()); + ClusterState state = clusterService.state(); + ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); + new AsyncIndexBySearchAction(bulkTask, logger, client, threadPool, request, scriptService, state, listener).start(); + } + + +/* if (slices > 1) { ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; parentTask.setSlices(slices); BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, @@ -75,7 +93,7 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, listener).start(); - } + }*/ }); } @@ -88,13 +106,13 @@ protected void doExecute(UpdateByQueryRequest request, ActionListener { - AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener) { this(task, logger, client, threadPool, request, scriptService, clusterState, listener, client.settings()); } - AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, + AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client, ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState, ActionListener listener, Settings settings) { super(task, logger, client, threadPool, request, scriptService, clusterState, listener, settings); @@ -113,7 +131,7 @@ protected boolean needsSourceDocumentVersions() { public BiFunction, ScrollableHitSource.Hit, RequestWrapper> buildScriptApplier() { Script script = mainRequest.getScript(); if (script != null) { - return new UpdateByQueryScriptApplier(task, scriptService, script, script.getParams()); + return new UpdateByQueryScriptApplier(worker, scriptService, script, script.getParams()); } return super.buildScriptApplier(); } @@ -133,9 +151,9 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) class UpdateByQueryScriptApplier extends ScriptApplier { - UpdateByQueryScriptApplier(WorkingBulkByScrollTask task, ScriptService scriptService, Script script, - Map params) { - super(task, scriptService, script, params); + UpdateByQueryScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, + Map params) { + super(taskWorker, scriptService, script, params); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java index 079c784342b2b..75f03ad1075a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java @@ -32,12 +32,14 @@ public abstract class AbstractAsyncBulkByScrollActionTestCase< Response extends BulkByScrollResponse> extends ESTestCase { protected ThreadPool threadPool; - protected WorkingBulkByScrollTask task; + protected BulkByScrollTask task; @Before public void setupForTest() { threadPool = new TestThreadPool(getTestName()); - task = new WorkingBulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID, null, Float.MAX_VALUE); + task = new BulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID); + task.setChild(null, Float.POSITIVE_INFINITY); + } @After From 6e49fdd2651c38c0bf9bb172d38d51f6a6c9fc16 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Fri, 21 Jul 2017 12:08:06 -0700 Subject: [PATCH 06/21] wip #24547 Production and test code compiles org.elasticsearch.index.reindex.AsyncBulkByScrollActionTests#testCancelWhileDelayedAfterScrollResponse and some others are broken --- .../reindex/ChildBulkByScrollWorker.java | 3 +- ...java => ChildBulkByScrollWorkerTests.java} | 39 ++++++++++--------- ...ava => ParentBulkByScrollWorkerTests.java} | 13 ++++--- .../reindex/AsyncBulkByScrollActionTests.java | 15 ++++--- .../TransportRethrottleActionTests.java | 11 +++--- 5 files changed, 46 insertions(+), 35 deletions(-) rename core/src/test/java/org/elasticsearch/index/reindex/{WorkingBulkByScrollTaskTests.java => ChildBulkByScrollWorkerTests.java} (88%) rename core/src/test/java/org/elasticsearch/index/reindex/{ParentBulkByScrollTaskTests.java => ParentBulkByScrollWorkerTests.java} (93%) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index c6c43672516b9..f94a947c4ee3a 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.FutureUtils; @@ -39,7 +40,7 @@ public class ChildBulkByScrollWorker implements SuccessfullyProcessed { - private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName()); + private static final Logger logger = Loggers.getLogger(ChildBulkByScrollWorker.class); private final BulkByScrollTask task; diff --git a/core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java similarity index 88% rename from core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java rename to core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java index 5d594d080b8b0..db25d736390db 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java @@ -46,12 +46,15 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class WorkingBulkByScrollTaskTests extends ESTestCase { - private WorkingBulkByScrollTask task; +public class ChildBulkByScrollWorkerTests extends ESTestCase { + private BulkByScrollTask task; + private ChildBulkByScrollWorker worker; @Before public void createTask() { - task = new WorkingBulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID, null, Float.POSITIVE_INFINITY); + task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); + task.setChild(null, Float.POSITIVE_INFINITY); + worker = task.getChildWorker(); } public void testBasicData() { @@ -78,7 +81,7 @@ public void testProgress() { assertEquals(noops, status.getNoops()); long totalHits = randomIntBetween(10, 1000); - task.setTotal(totalHits); + worker.setTotal(totalHits); for (long p = 0; p < totalHits; p++) { status = task.getStatus(); assertEquals(totalHits, status.getTotal()); @@ -91,28 +94,28 @@ public void testProgress() { if (randomBoolean()) { created++; - task.countCreated(); + worker.countCreated(); } else if (randomBoolean()) { updated++; - task.countUpdated(); + worker.countUpdated(); } else { deleted++; - task.countDeleted(); + worker.countDeleted(); } if (rarely()) { versionConflicts++; - task.countVersionConflict(); + worker.countVersionConflict(); } if (rarely()) { batch++; - task.countBatch(); + worker.countBatch(); } if (rarely()) { noops++; - task.countNoop(); + worker.countNoop(); } } status = task.getStatus(); @@ -139,7 +142,7 @@ public void testDelayAndRethrottle() throws IOException, InterruptedException { * each time. */ float originalRequestsPerSecond = (float) randomDoubleBetween(1, 10000, true); - task.rethrottle(originalRequestsPerSecond); + worker.rethrottle(originalRequestsPerSecond); TimeValue maxDelay = timeValueSeconds(between(1, 5)); assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L)); int batchSizeForMaxDelay = (int) (maxDelay.seconds() * originalRequestsPerSecond); @@ -151,7 +154,7 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman } }; try { - task.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() { + worker.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() { @Override protected void doRun() throws Exception { boolean oldValue = done.getAndSet(true); @@ -172,7 +175,7 @@ public void onFailure(Exception e) { int rethrottles = 0; while (false == done.get()) { float requestsPerSecond = (float) randomDoubleBetween(0, originalRequestsPerSecond * 2, true); - task.rethrottle(requestsPerSecond); + worker.rethrottle(requestsPerSecond); rethrottles += 1; } logger.info("Rethrottled [{}] times", rethrottles); @@ -237,7 +240,7 @@ public Void get(long timeout, TimeUnit unit) throws InterruptedException, Execut }; try { // Have the task use the thread pool to delay a task that does nothing - task.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() { + worker.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() { @Override protected void doRun() throws Exception { } @@ -254,12 +257,12 @@ public void onFailure(Exception e) { } public void testPerfectlyThrottledBatchTime() { - task.rethrottle(Float.POSITIVE_INFINITY); - assertThat((double) task.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f)); + worker.rethrottle(Float.POSITIVE_INFINITY); + assertThat((double) worker.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f)); int total = between(0, 1000000); - task.rethrottle(1); - assertThat((double) task.perfectlyThrottledBatchTime(total), + worker.rethrottle(1); + assertThat((double) worker.perfectlyThrottledBatchTime(total), closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1))); } } diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java similarity index 93% rename from core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java rename to core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java index fd6a992800193..d0894327b27c3 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -33,15 +34,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -public class ParentBulkByScrollTaskTests extends ESTestCase { +public class ParentBulkByScrollWorkerTests extends ESTestCase { private int slices; - private ParentBulkByScrollTask task; + private BulkByScrollTask task; + private ParentBulkByScrollWorker worker; @Before public void createTask() { slices = between(2, 50); - task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null); - task.setSlices(slices); + task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); + task.setParent(slices); + worker = task.getParentWorker(); } public void testBasicData() { @@ -91,7 +94,7 @@ public void testProgress() { @SuppressWarnings("unchecked") ActionListener listener = slice < slices - 1 ? neverCalled() : mock(ActionListener.class); - task.onSliceResponse(listener, slice, + worker.onSliceResponse(listener, slice, new BulkByScrollResponse(timeValueMillis(10), sliceStatus, emptyList(), emptyList(), false)); status = task.getStatus(); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 45b2d4712e39f..ee51d0b43449b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -124,7 +124,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase { private PlainActionFuture listener; private String scrollId; private TaskManager taskManager; - private WorkingBulkByScrollTask testTask; + private BulkByScrollTask testTask; + private ChildBulkByScrollWorker worker; private Map expectedHeaders = new HashMap<>(); private DiscoveryNode localNode; private TaskId taskId; @@ -141,7 +142,9 @@ public void setupForTest() { listener = new PlainActionFuture<>(); scrollId = null; taskManager = new TaskManager(Settings.EMPTY); - testTask = (WorkingBulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest); + testTask = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest); + testTask.setChild(null, testRequest.getRequestsPerSecond()); // todo this is awkward - is there a way to set this stuff in the task or somewhere else + worker = testTask.getChildWorker(); localNode = new DiscoveryNode("thenode", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); taskId = new TaskId(localNode.getId(), testTask.getId()); @@ -309,7 +312,7 @@ public void testBulkResponseSetsLotsOfStatus() { * Mimicks a ThreadPool rejecting execution of the task. */ public void testThreadPoolRejectionsAbortRequest() throws Exception { - testTask.rethrottle(1); + worker.rethrottle(1); setupClient(new TestThreadPool(getTestName()) { @Override public ScheduledFuture schedule(TimeValue delay, String name, Runnable command) { @@ -439,7 +442,7 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman firstSearchRequest.scroll(timeValueSeconds(10)); // Set throttle to 1 request per second to make the math simpler - testTask.rethrottle(1f); + worker.rethrottle(1f); // Make the last batch look nearly instant but have 100 documents TimeValue lastBatchStartTime = timeValueNanos(System.nanoTime()); TimeValue now = timeValueNanos(lastBatchStartTime.nanos() + 1); @@ -459,7 +462,7 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman assertEquals(99, capturedDelay.get().seconds()); } else { // Let's rethrottle between the starting the scroll and getting the response - testTask.rethrottle(10f); + worker.rethrottle(10f); client.lastScroll.get().listener.onResponse(searchResponse); // The delay uses the new throttle assertEquals(9, capturedDelay.get().seconds()); @@ -624,7 +627,7 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman long total = randomIntBetween(0, Integer.MAX_VALUE); ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null); // Use a long delay here so the test will time out if the cancellation doesn't reschedule the throttled task - testTask.rethrottle(1); + worker.rethrottle(1); simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1000, response); // Now that we've got our cancel we'll just verify that it all came through all right diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java index 26b2a6e502ee5..0511eef525837 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java @@ -48,13 +48,13 @@ public class TransportRethrottleActionTests extends ESTestCase { private int slices; - private ParentBulkByScrollTask task; + private BulkByScrollTask task; @Before public void createTask() { slices = between(2, 50); - task = new ParentBulkByScrollTask(1, "test_type", "test_action", "test", null); - task.setSlices(slices); + task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); + task.setParent(slices); } /** @@ -114,7 +114,7 @@ public void testRethrottleWithSomeSucceeded() { List sliceStatuses = new ArrayList<>(slices); for (int i = 0; i < succeeded; i++) { BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.onSliceResponse(neverCalled(), i, + task.getParentWorker().onSliceResponse(neverCalled(), i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); sliceStatuses.add(new BulkByScrollTask.StatusOrException(status)); } @@ -135,7 +135,8 @@ public void testRethrottleWithAllSucceeded() { @SuppressWarnings("unchecked") ActionListener listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class); BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); + task.getParentWorker().onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), + emptyList(), false)); if (i == slices - 1) { // The whole thing succeeded so we should have got the success captureResponse(BulkByScrollResponse.class, listener).getStatus(); From 3c90d9d5f78fc90b2de79c02714cb044a6431d35 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 25 Jul 2017 09:02:12 -0700 Subject: [PATCH 07/21] wip #24547 Tests with rethrottling still broken --- .../support/tasks/TransportTasksAction.java | 3 +++ .../index/reindex/BulkByScrollTask.java | 18 +++++++++++++++++- .../index/reindex/ChildBulkByScrollWorker.java | 7 +++++++ .../reindex/ParentBulkByScrollWorker.java | 2 +- .../reindex/TransportRethrottleAction.java | 5 +++++ .../reindex/AsyncBulkByScrollActionTests.java | 2 +- .../index/reindex/RethrottleTests.java | 15 +++++++++++++++ 7 files changed, 49 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index 35b2b41dfda6e..487ea85e24b93 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -149,6 +149,9 @@ private void respondIfFinished() { } }; try { + //todo remove + Task task = tasks.get(taskIndex); + logger.warn("[{}] TASKTYPE {}", task.getId(), task.getClass()); taskOperation(request, tasks.get(taskIndex), taskListener); } catch (Exception e) { taskListener.onFailure(e); diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 22d700a153779..160129f631b5e 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -34,6 +34,7 @@ import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -64,7 +65,11 @@ public BulkByScrollTask.Status getStatus() { return childWorker.getStatus(); } - throw new IllegalStateException("This task's worker is not set"); + return emptyStatus(); + } + + private BulkByScrollTask.Status emptyStatus() { + return new Status(Collections.emptyList(), getReasonCancelled()); } public boolean isParent() { @@ -105,6 +110,17 @@ public ChildBulkByScrollWorker getChildWorker() { return childWorker; } + @Override + public void onCancelled() { + if (isParent()) { + // do nothing + } else if (isChild()) { + childWorker.handleCancel(); + } else { + throw new IllegalStateException("This task's worker is not set"); + } + } + @Override public boolean shouldCancelChildrenOnCancellation() { return true; diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index f94a947c4ee3a..8540a1529dd71 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -87,6 +87,11 @@ public BulkByScrollTask.Status getStatus() { task.getReasonCancelled(), throttledUntil()); } + public void handleCancel() { + // Drop the throttle to 0, immediately rescheduling any throttle operation so it will wake up and cancel itself. + rethrottle(Float.POSITIVE_INFINITY); + } + public void setTotal(long totalHits) { total.set(totalHits); } @@ -196,6 +201,7 @@ private void setRequestsPerSecond(float requestsPerSecond) { } public void rethrottle(float newRequestsPerSecond) { + logger.warn("[{}]: STARTING CHILD RETHROTTLE", task.getId()); synchronized (delayedPrepareBulkRequestReference) { logger.debug("[{}]: rethrottling to [{}] requests per second", task.getId(), newRequestsPerSecond); setRequestsPerSecond(newRequestsPerSecond); @@ -209,6 +215,7 @@ public void rethrottle(float newRequestsPerSecond) { this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond)); } + logger.warn("[{}]:22 FINISH CHILD RETHROTTLE HANDLING TASK", task.getId()); } class DelayedPrepareBulkRequest { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java index 447b0dd9e7383..f9ed1583d12fe 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java @@ -55,7 +55,7 @@ public int getSlices() { return slices; } - public BulkByScrollTask.Status getStatus() { // moved to parent worker + public BulkByScrollTask.Status getStatus() { // We only have access to the statuses of requests that have finished so we return them List statuses = Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()]); addResultsToList(statuses); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index 4c5282c4312e9..f63d37a47b1ed 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -54,7 +54,9 @@ public TransportRethrottleAction(Settings settings, ThreadPool threadPool, Clust @Override protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, ActionListener listener) { + logger.warn("[{}]: RETHROTTLE start taskOperation", task.getId()); rethrottle(logger, clusterService.localNode().getId(), client, task, request.getRequestsPerSecond(), listener); + logger.warn("[{}]: RETHROTTLE finish taskOperation", task.getId()); } static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, @@ -62,14 +64,17 @@ static void rethrottle(Logger logger, String localNodeId, Client client, BulkByS if (task.isChild()) { rethrottleChildTask(logger, localNodeId, task, newRequestsPerSecond, listener); + logger.warn("[{}]: FINISH CHILD INIT RETHROTTLE", task.getId()); return; } if (task.isParent()) { rethrottleParentTask(logger, localNodeId, client, task, newRequestsPerSecond, listener); + logger.warn("[{}]: FINISH PARENT INIT RETHROTTLE", task.getId()); return; } + logger.warn("[{}]: THROW EXCEPTION RETHROTTLE", task.getId()); throw new IllegalArgumentException("task [" + task.getId() + "] must be set as a child or parent"); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index ee51d0b43449b..78f734d036548 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -590,7 +590,7 @@ public void testCancelBeforeRefreshAndFinish() throws Exception { assertNull("No refresh was attempted", client.lastRefreshRequest.get()); } - /** + /* * Tests that we can cancel the request during its throttling delay. This can't use {@link #cancelTaskCase(Consumer)} because it needs * to send the request un-canceled and cancel it at a specific time. */ diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index 4af0dd3797a11..e1cebae6a2154 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -82,17 +82,23 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a } indexRandom(true, docs); + logger.info("Executing request"); + // Start a request that will never finish unless we rethrottle it request.setRequestsPerSecond(.000001f); // Throttle "forever" request.source().setSize(1); // Make sure we use multiple batches ActionFuture responseListener = request.execute(); + logger.info("Finished executing request"); + TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices().number()); TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId(); if (request.request().getSlices().number() == 1) { + logger.info("There are no child tasks"); assertThat(taskGroupToRethrottle.getChildTasks(), empty()); } else { + logger.info("Asserting child tasks are running"); // There should be a sane number of child tasks running assertThat(taskGroupToRethrottle.getChildTasks(), hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices().number())))); @@ -108,12 +114,17 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a }); } + logger.info("Rethrottling tasks"); + // Now rethrottle it so it'll finish float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000; // No throttle or "very fast" ListTasksResponse rethrottleResponse = rethrottle().setTaskId(taskToRethrottle).setRequestsPerSecond(newRequestsPerSecond).get(); rethrottleResponse.rethrowFailures("Rethrottle"); assertThat(rethrottleResponse.getTasks(), hasSize(1)); BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus(); + + logger.info("Finished executing rethrottle task"); + // Now check the resulting requests per second. if (request.request().getSlices().number() == 1) { // If there is a single slice it should match perfectly @@ -161,10 +172,14 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f); } + logger.info("Asserted a bunch of stuff"); + // Now the response should come back quickly because we've rethrottled the request BulkByScrollResponse response = responseListener.get(); assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", response.getBatches(), greaterThanOrEqualTo(request.request().getSlices().number())); + + logger.info("Test case is done"); } private TaskGroup findTaskToRethrottle(String actionName, int sliceCount) { From 1814f4f7d1fd418a2d90e3addc982d94d2de9ebc Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 25 Jul 2017 17:47:06 -0700 Subject: [PATCH 08/21] wip #24547 --- .../elasticsearch/index/reindex/ChildBulkByScrollWorker.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index 8540a1529dd71..6e7a52fcaccb0 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.reindex; import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -215,7 +214,7 @@ public void rethrottle(float newRequestsPerSecond) { this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond)); } - logger.warn("[{}]:22 FINISH CHILD RETHROTTLE HANDLING TASK", task.getId()); + logger.warn("[{}]: FINISH CHILD RETHROTTLE HANDLING TASK", task.getId()); } class DelayedPrepareBulkRequest { From dacc62ac72e2894ad854e069bcc41bac48372274 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 31 Jul 2017 11:09:19 -0700 Subject: [PATCH 09/21] wip #24547 fix tests All tests pass now for core and reindex module --- .../index/reindex/BulkByScrollResponse.java | 3 +- .../index/reindex/ParentBulkByScrollTask.java | 192 ---------- .../index/reindex/SlicesCount.java | 4 +- .../reindex/WorkingBulkByScrollTask.java | 339 ------------------ .../search/slice/SliceBuilderTests.java | 12 +- .../reindex/TransportRethrottleAction.java | 3 +- .../reindex/DeleteByQueryBasicTests.java | 26 +- .../index/reindex/ReindexBasicTests.java | 7 +- .../index/reindex/RethrottleTests.java | 2 +- .../reindex/UpdateByQueryBasicTests.java | 34 +- .../test/delete_by_query/20_validation.yml | 2 +- .../test/reindex/20_validation.yml | 2 +- .../test/update_by_query/20_validation.yml | 2 +- 13 files changed, 70 insertions(+), 558 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java delete mode 100644 core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java index 400baf7b9e2c6..0b70c4e7a7b29 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java @@ -190,7 +190,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder.append("BulkIndexByScrollResponse["); + builder.append(getClass().getSimpleName()); + builder.append("["); builder.append("took=").append(took).append(','); builder.append("timed_out=").append(timedOut).append(','); status.innerToString(builder); diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java deleted file mode 100644 index af69419d8a861..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.reindex; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.tasks.TaskInfo; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.util.Collections.unmodifiableList; - -/** - * Task for parent bulk by scroll requests that have sub-workers. todo remove - */ -public class ParentBulkByScrollTask extends BulkByScrollTask { - public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { - super(id, type, action, description, parentTaskId); - } - /** - * Holds the responses as they come back. This uses {@link Tuple} as an "Either" style holder where only the response or the exception - * is set. - private AtomicArray results; - private AtomicInteger counter; - - // todo it may still make sense to store the original slices value in here even if it's not used - // todo either enforce slices being set before calling task methods or separate that out into another class - // todo probably want to look at the tests for this class and make sure the slices set behavior is enforced - - // todo this class may not be thread safe because the accesses to results and counter are not synchronized - // e.g. it's possible to read all task results finished and runningSubtasks != 0 at the same time - - public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { - super(id, type, action, description, parentTaskId); - } - - public void setSlices(int slices) { - if (slices < 1) { - throw new IllegalArgumentException("Slices must be at least one"); - } - if (isSlicesSet()) { - throw new IllegalStateException("Slices are already set"); - } - results = new AtomicArray<>(slices); - counter = new AtomicInteger(slices); - } - - public boolean isSlicesSet() { - return results != null && counter != null; - } - - @Override - public void rethrottle(float newRequestsPerSecond) { - // Nothing to do because all rethrottling is done on slice sub tasks. - } - - @Override - public Status getStatus() { - // We only have access to the statuses of requests that have finished so we return them - List statuses = Arrays.asList(new StatusOrException[results.length()]); - addResultsToList(statuses); - return new Status(unmodifiableList(statuses), getReasonCancelled()); - } - - @Override - public int runningSliceSubTasks() { - return counter.get(); - } - - @Override - public TaskInfo getInfoGivenSliceInfo(String localNodeId, List sliceInfo) { - *//* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the - * finished status because we don't expect them to change after the task is finished. *//* - List sliceStatuses = Arrays.asList(new StatusOrException[results.length()]); - for (TaskInfo t : sliceInfo) { - Status status = (Status) t.getStatus(); - sliceStatuses.set(status.getSliceId(), new StatusOrException(status)); - } - addResultsToList(sliceStatuses); - Status status = new Status(sliceStatuses, getReasonCancelled()); - return taskInfo(localNodeId, getDescription(), status); - } - - private void addResultsToList(List sliceStatuses) { - for (Result t : results.asList()) { - if (t.response != null) { - sliceStatuses.set(t.sliceId, new StatusOrException(t.response.getStatus())); - } else { - sliceStatuses.set(t.sliceId, new StatusOrException(t.failure)); - } - } - } - - *//** - * Record a response from a slice and respond to the listener if the request is finished. - *//* - public void onSliceResponse(ActionListener listener, int sliceId, BulkByScrollResponse response) { - results.setOnce(sliceId, new Result(sliceId, response)); - *//* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we - * were fairly sure they had a while left to go. *//* - recordSliceCompletionAndRespondIfAllDone(listener); - } - - - // todo this is not an atomic operation: you can read all the tasks results finished while counter is still not 0 - *//* - * e.g. - * there are two child tasks - * - * thread A enters onSliceResponse - * thread A sets a result for child task 1 - * thread B enters onSliceResponse - * thread B enters a result for task 2 - * at this point all tasks are done - * thread B decrements counter - * thread A decrements counter and now thinks it's the last task even though it's not - *//* - *//** - * Record a failure from a slice and respond to the listener if the request is finished. - *//* - public void onSliceFailure(ActionListener listener, int sliceId, Exception e) { - results.setOnce(sliceId, new Result(sliceId, e)); - recordSliceCompletionAndRespondIfAllDone(listener); - // TODO cancel when a slice fails? - } - - private void recordSliceCompletionAndRespondIfAllDone(ActionListener listener) { - if (counter.decrementAndGet() != 0) { - return; - } - List responses = new ArrayList<>(results.length()); - Exception exception = null; - for (Result t : results.asList()) { - if (t.response == null) { - assert t.failure != null : "exception shouldn't be null if value is null"; - if (exception == null) { - exception = t.failure; - } else { - exception.addSuppressed(t.failure); - } - } else { - assert t.failure == null : "exception should be null if response is not null"; - responses.add(t.response); - } - } - if (exception == null) { - listener.onResponse(new BulkByScrollResponse(responses, getReasonCancelled())); - } else { - listener.onFailure(exception); - } - } - - private static final class Result { - final BulkByScrollResponse response; - final int sliceId; - final Exception failure; - - private Result(int sliceId, BulkByScrollResponse response) { - this.sliceId = sliceId; - this.response = response; - failure = null; - } - - private Result(int sliceId, Exception failure) { - this.sliceId = sliceId; - this.failure = failure; - response = null; - } - }*/ -} diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java index 92a5e6323fdda..bdffac8e2fc72 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java @@ -51,7 +51,7 @@ public SlicesCount(StreamInput stream) throws IOException { public static SlicesCount of(int count) { if (count < 1) { - throw new IllegalArgumentException("Slice count must be at least 1"); + throw new IllegalArgumentException("[slices] must be at least 1"); } return new SlicesCount(count); } @@ -67,7 +67,7 @@ public static SlicesCount parse(String slicesString) { int slicesNumber = Integer.parseInt(slicesString); return of(slicesNumber); } catch (NumberFormatException e) { - throw new IllegalArgumentException("Slices must be a positive integer or the string \"auto\""); + throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\""); } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java deleted file mode 100644 index 35c86170a646b..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java +++ /dev/null @@ -1,339 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.reindex; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.common.util.concurrent.FutureUtils; -import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.tasks.TaskInfo; -import org.elasticsearch.threadpool.ThreadPool; - -import java.util.List; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import static java.lang.Math.max; -import static java.lang.Math.round; -import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; - -/** - * todo remove. - */ -public class WorkingBulkByScrollTask extends BulkByScrollTask implements SuccessfullyProcessed { - public WorkingBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { - super(id, type, action, description, parentTaskId); - } - - @Override - public long getUpdated() { - return 0; - } - - @Override - public long getCreated() { - return 0; - } - - @Override - public long getDeleted() { - return 0; - } - /*private static final Logger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName()); - - *//** - * The id of the slice that this task is processing or {@code null} if this task isn't for a sliced request. - *//* - private final Integer sliceId; - *//** - * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents - * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived. - *//* - private final AtomicLong total = new AtomicLong(0); - private final AtomicLong updated = new AtomicLong(0); - private final AtomicLong created = new AtomicLong(0); - private final AtomicLong deleted = new AtomicLong(0); - private final AtomicLong noops = new AtomicLong(0); - private final AtomicInteger batch = new AtomicInteger(0); - private final AtomicLong versionConflicts = new AtomicLong(0); - private final AtomicLong bulkRetries = new AtomicLong(0); - private final AtomicLong searchRetries = new AtomicLong(0); - private final AtomicLong throttledNanos = new AtomicLong(); - *//** - * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX - * style variables but there isn't an AtomicFloat so we just use a volatile. - *//* - private volatile float requestsPerSecond; - *//** - * Reference to any the last delayed prepareBulkRequest call. Used during rethrottling and canceling to reschedule the request. - *//* - private final AtomicReference delayedPrepareBulkRequestReference = new AtomicReference<>(); - - public WorkingBulkByScrollTask(long id, String type, String action, String description, TaskId parentTask, Integer sliceId, - float requestsPerSecond) { - super(id, type, action, description, parentTask); - this.sliceId = sliceId; - setRequestsPerSecond(requestsPerSecond); - } - - @Override - public Status getStatus() { - return new Status(sliceId, total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), - noops.get(), bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), - getReasonCancelled(), throttledUntil()); - } - - @Override - protected void onCancelled() { - Drop the throttle to 0, immediately rescheduling any throttled - * operation so it will wake up and cancel itself. - rethrottle(Float.POSITIVE_INFINITY); - } - - @Override - public int runningSliceSubTasks() { - return 0; - } - - @Override - public TaskInfo getInfoGivenSliceInfo(String localNodeId, List sliceInfo) { - throw new UnsupportedOperationException("This is only supported by " + ParentBulkByScrollTask.class.getName() + "."); - } - - TimeValue throttledUntil() { - DelayedPrepareBulkRequest delayed = delayedPrepareBulkRequestReference.get(); - if (delayed == null) { - return timeValueNanos(0); - } - if (delayed.future == null) { - return timeValueNanos(0); - } - return timeValueNanos(max(0, delayed.future.getDelay(TimeUnit.NANOSECONDS))); - } - - public void setTotal(long totalHits) { - total.set(totalHits); - } - - public void countBatch() { - batch.incrementAndGet(); - } - - public void countNoop() { - noops.incrementAndGet(); - } - - @Override - public long getCreated() { - return created.get(); - } - - public void countCreated() { - created.incrementAndGet(); - } - - @Override - public long getUpdated() { - return updated.get(); - } - - public void countUpdated() { - updated.incrementAndGet(); - } - - @Override - public long getDeleted() { - return deleted.get(); - } - - public void countDeleted() { - deleted.incrementAndGet(); - } - - public void countVersionConflict() { - versionConflicts.incrementAndGet(); - } - - public void countBulkRetry() { - bulkRetries.incrementAndGet(); - } - - public void countSearchRetry() { - searchRetries.incrementAndGet(); - } - - float getRequestsPerSecond() { - return requestsPerSecond; - } - - *//** - * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be - * rescheduled over and over again. - *//* - public void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize, - AbstractRunnable prepareBulkRequestRunnable) { - // Synchronize so we are less likely to schedule the same request twice. - synchronized (delayedPrepareBulkRequestReference) { - TimeValue delay = throttleWaitTime(lastBatchStartTime, timeValueNanos(System.nanoTime()), lastBatchSize); - logger.debug("[{}]: preparing bulk request for [{}]", getId(), delay); - delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(), - delay, new RunOnce(prepareBulkRequestRunnable))); - } - } - - public TimeValue throttleWaitTime(TimeValue lastBatchStartTime, TimeValue now, int lastBatchSize) { - long earliestNextBatchStartTime = now.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize); - return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())); - } - - *//** - * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing. - *//* - float perfectlyThrottledBatchTime(int lastBatchSize) { - if (requestsPerSecond == Float.POSITIVE_INFINITY) { - return 0; - } - // requests - // ------------------- == seconds - // request per seconds - float targetBatchTimeInSeconds = lastBatchSize / requestsPerSecond; - // nanoseconds per seconds * seconds == nanoseconds - return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds; - } - - private void setRequestsPerSecond(float requestsPerSecond) { - if (requestsPerSecond <= 0) { - throw new IllegalArgumentException("requests per second must be more than 0 but was [" + requestsPerSecond + "]"); - } - this.requestsPerSecond = requestsPerSecond; - } - - - @Override - public void rethrottle(float newRequestsPerSecond) { - synchronized (delayedPrepareBulkRequestReference) { - logger.debug("[{}]: rethrottling to [{}] requests per second", getId(), newRequestsPerSecond); - setRequestsPerSecond(newRequestsPerSecond); - - DelayedPrepareBulkRequest delayedPrepareBulkRequest = this.delayedPrepareBulkRequestReference.get(); - if (delayedPrepareBulkRequest == null) { - // No request has been queued so nothing to reschedule. - logger.debug("[{}]: skipping rescheduling because there is no scheduled task", getId()); - return; - } - - this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond)); - } - } - - class DelayedPrepareBulkRequest { - private final ThreadPool threadPool; - private final AbstractRunnable command; - private final float requestsPerSecond; - private final ScheduledFuture future; - - DelayedPrepareBulkRequest(ThreadPool threadPool, float requestsPerSecond, TimeValue delay, AbstractRunnable command) { - this.threadPool = threadPool; - this.requestsPerSecond = requestsPerSecond; - this.command = command; - this.future = threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - throttledNanos.addAndGet(delay.nanos()); - command.run(); - } - - @Override - public void onFailure(Exception e) { - command.onFailure(e); - } - }); - } - - DelayedPrepareBulkRequest rethrottle(float newRequestsPerSecond) { - if (newRequestsPerSecond < requestsPerSecond) { - *//* The user is attempting to slow the request down. We'll let the - * change in throttle take effect the next time we delay - * prepareBulkRequest. We can't just reschedule the request further - * out in the future because the bulk context might time out. *//* - logger.debug("[{}]: skipping rescheduling because the new throttle [{}] is slower than the old one [{}]", getId(), - newRequestsPerSecond, requestsPerSecond); - return this; - } - - long remainingDelay = future.getDelay(TimeUnit.NANOSECONDS); - // Actually reschedule the task - if (false == FutureUtils.cancel(future)) { - // Couldn't cancel, probably because the task has finished or been scheduled. Either way we have nothing to do here. - logger.debug("[{}]: skipping rescheduling because we couldn't cancel the task", getId()); - return this; - } - - *//* Strangely enough getting here doesn't mean that you actually - * cancelled the request, just that you probably did. If you stress - * test it you'll find that requests sneak through. So each request - * is given a runOnce boolean to prevent that. *//* - TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond); - logger.debug("[{}]: rescheduling for [{}] in the future", getId(), newDelay); - return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command); - } - - *//** - * Scale back remaining delay to fit the new delay. - *//* - TimeValue newDelay(long remainingDelay, float newRequestsPerSecond) { - if (remainingDelay < 0) { - return timeValueNanos(0); - } - return timeValueNanos(round(remainingDelay * requestsPerSecond / newRequestsPerSecond)); - } - } - - *//** - * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times. - * Without it the command would be run multiple times. - *//* - private static class RunOnce extends AbstractRunnable { - private final AtomicBoolean hasRun = new AtomicBoolean(false); - private final AbstractRunnable delegate; - - RunOnce(AbstractRunnable delegate) { - this.delegate = delegate; - } - - @Override - protected void doRun() throws Exception { - if (hasRun.compareAndSet(false, true)) { - delegate.run(); - } - } - - @Override - public void onFailure(Exception e) { - delegate.onFailure(e); - } - }*/ -} diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index a70a6f30be37a..7ea3dd2094b64 100644 --- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -114,21 +114,21 @@ public void testFromXContent() throws Exception { public void testInvalidArguments() throws Exception { Exception e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", -1, 10)); - assertEquals(e.getMessage(), "id must be greater than or equal to 0"); + assertEquals("id must be greater than or equal to 0", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, -1)); - assertEquals(e.getMessage(), "max must be greater than 0"); + assertEquals("max must be greater than 1", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 0)); - assertEquals(e.getMessage(), "max must be greater than 0"); + assertEquals("max must be greater than 1", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 5)); - assertEquals(e.getMessage(), "max must be greater than id"); + assertEquals("max must be greater than id", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 1000, 1000)); - assertEquals(e.getMessage(), "max must be greater than id"); + assertEquals("max must be greater than id", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 1001, 1000)); - assertEquals(e.getMessage(), "max must be greater than id"); + assertEquals("max must be greater than id", e.getMessage()); } public void testToFilter() throws IOException { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index f63d37a47b1ed..6b2c976c0fe80 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -85,7 +85,7 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie if (runningSubtasks > 0) { RethrottleRequest subRequest = new RethrottleRequest(); subRequest.setRequestsPerSecond(newRequestsPerSecond / runningSubtasks); - subRequest.setParentTask(new TaskId(localNodeId, task.getId())); + subRequest.setParentTaskId(new TaskId(localNodeId, task.getId())); logger.debug("rethrottling children of task [{}] to [{}] requests per second", task.getId(), subRequest.getRequestsPerSecond()); client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap( @@ -96,6 +96,7 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie listener::onFailure)); } else { logger.debug("children of task [{}] are already finished, nothing to rethrottle", task.getId()); + listener.onResponse(task.taskInfo(localNodeId, true)); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index 03e46af73c8a1..e3e69a038ab44 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -262,17 +262,31 @@ public void testSlicesAuto() throws Exception { assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); NumShards numShards = getNumShards("test"); - logger.error("SHARDSCOUNT {}", numShards.numPrimaries); + int numSlicesExpected = numShards.numPrimaries > 1 + ? numShards.numPrimaries + : 0; // Deletes the two docs that matches "foo:a" - assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.AUTO).get(), - matcher().deleted(2).slices(hasSize(numShards.numPrimaries))); + assertThat( + deleteByQuery() + .source("test") + .filter(termQuery("foo", "a")) + .refresh(true) + .setSlices(SlicesCount.AUTO).get(), + matcher() + .deleted(2) + .slices(hasSize(numSlicesExpected))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); // Delete remaining docs - DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) - .setSlices(SlicesCount.AUTO); - assertThat(request.get(), matcher().deleted(5).slices(hasSize(numShards.numPrimaries))); + assertThat(deleteByQuery() + .source("test") + .filter(QueryBuilders.matchAllQuery()) + .refresh(true) + .setSlices(SlicesCount.AUTO).get(), + matcher() + .deleted(5) + .slices(hasSize(numSlicesExpected))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index 5688252326063..d179316a1b8ec 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -128,10 +128,13 @@ public void testCopyManyWithSlicesAuto() throws Exception { assertHitCount(client().prepareSearch("source").setSize(0).get(), max); NumShards numShards = getNumShards("source"); + int numSlicesExpected = numShards.numPrimaries > 1 + ? numShards.numPrimaries + : 0; ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(SlicesCount.AUTO); copy.source().setSize(5); - assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(numShards.numPrimaries))); + assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(numSlicesExpected))); assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max); int half = max / 2; @@ -139,7 +142,7 @@ public void testCopyManyWithSlicesAuto() throws Exception { copy.source().setSize(5); copy.size(half); BulkByScrollResponse response = copy.get(); - assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(numShards.numPrimaries))); + assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(numSlicesExpected))); assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated()); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index e1cebae6a2154..cbe1758457af0 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -114,7 +114,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a }); } - logger.info("Rethrottling tasks"); + logger.info("Rethrottling task [{}]", taskToRethrottle.getId()); // Now rethrottle it so it'll finish float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000; // No throttle or "very fast" diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index fb7ef4a026cef..8e9112898635e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -102,21 +102,45 @@ public void testSlicesAuto() throws Exception { assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); NumShards numShards = getNumShards("test"); + int numSlicesExpected = numShards.numPrimaries > 1 + ? numShards.numPrimaries + : 0; // Reindex all the docs - assertThat(updateByQuery().source("test").refresh(true).setSlices(SlicesCount.AUTO).get(), matcher().updated(4).slices(hasSize(numShards.numPrimaries))); + assertThat( + updateByQuery() + .source("test") + .refresh(true) + .setSlices(SlicesCount.AUTO).get(), + matcher() + .updated(4) + .slices(hasSize(numSlicesExpected))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now none of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(SlicesCount.AUTO).refresh(true).get(), - matcher().updated(0).slices(hasSize(numShards.numPrimaries))); + assertThat( + updateByQuery() + .source("test") + .filter(termQuery("foo", "no_match")) + .setSlices(SlicesCount.AUTO) + .refresh(true).get(), + matcher() + .updated(0) + .slices(hasSize(numSlicesExpected))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now half of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.AUTO).get(), - matcher().updated(2).slices(hasSize(numShards.numPrimaries))); + assertThat( + updateByQuery() + .source("test") + .filter(termQuery("foo", "a")) + .refresh(true) + .setSlices(SlicesCount.AUTO).get(), + matcher() + .updated(2) + .slices(hasSize(numSlicesExpected))); assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion()); diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml index 9daf1502a362f..ba837bf655eeb 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml @@ -126,7 +126,7 @@ --- "junk in slices fails": - do: - catch: /Failed to parse int parameter \[slices\] with value \[junk\]/ + catch: /\[slices\] must be a positive integer or the string "auto"/ delete_by_query: slices: junk index: test diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml index b64eaac7dec22..c6cbc647943fe 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml @@ -275,7 +275,7 @@ --- "junk in slices fails": - do: - catch: /Failed to parse int parameter \[slices\] with value \[junk\]/ + catch: /\[slices\] must be a positive integer or the string "auto"/ reindex: slices: junk body: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml index 8f8d492df3aa0..35b212c854ee6 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml @@ -108,7 +108,7 @@ --- "junk in slices fails": - do: - catch: /Failed to parse int parameter \[slices\] with value \[junk\]/ + catch: /\[slices\] must be a positive integer or the string "auto"/ update_by_query: slices: junk index: test From 0f0904d94e7996f98ffbc3163ae896655e93df2b Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 31 Jul 2017 15:17:09 -0700 Subject: [PATCH 10/21] wip #24547 cleanup logging and docs --- .../support/tasks/TransportTasksAction.java | 3 -- .../reindex/AbstractBulkByScrollRequest.java | 25 +++------ .../AbstractBulkByScrollRequestBuilder.java | 2 +- .../index/reindex/BulkByScrollTask.java | 35 +++++++++++- .../reindex/ChildBulkByScrollWorker.java | 5 +- .../reindex/ParentBulkByScrollWorker.java | 22 ++++++-- .../reindex/{SlicesCount.java => Slices.java} | 39 ++++++++++---- .../AbstractBulkByScrollRequestTestCase.java | 10 ++-- .../index/reindex/ReindexRequestTests.java | 4 +- .../AbstractBaseReindexRestHandler.java | 4 +- .../BulkByScrollParallelizationHelper.java | 14 +++-- .../reindex/TransportDeleteByQueryAction.java | 14 ----- .../index/reindex/TransportReindexAction.java | 17 ------ .../reindex/TransportRethrottleAction.java | 5 -- .../reindex/TransportUpdateByQueryAction.java | 13 ----- .../reindex/AsyncBulkByScrollActionTests.java | 4 +- .../index/reindex/CancelTests.java | 6 +-- .../reindex/DeleteByQueryBasicTests.java | 12 ++--- .../index/reindex/ReindexBasicTests.java | 10 ++-- .../index/reindex/RethrottleTests.java | 54 ++++++++++--------- .../index/reindex/RoundTripTests.java | 18 ++++--- .../reindex/UpdateByQueryBasicTests.java | 14 ++--- 22 files changed, 174 insertions(+), 156 deletions(-) rename core/src/main/java/org/elasticsearch/index/reindex/{SlicesCount.java => Slices.java} (73%) diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java index 487ea85e24b93..35b2b41dfda6e 100644 --- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java @@ -149,9 +149,6 @@ private void respondIfFinished() { } }; try { - //todo remove - Task task = tasks.get(taskIndex); - logger.warn("[{}] TASKTYPE {}", task.getId(), task.getClass()); taskOperation(request, tasks.get(taskIndex), taskListener); } catch (Exception e) { taskListener.onFailure(e); diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 0ecae6da9ddae..312d9de7d64ad 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -102,7 +102,7 @@ public abstract class AbstractBulkByScrollRequest 1) { todo remove this - return new ParentBulkByScrollTask(id, type, action, getDescription(), parentTaskId); - } - *//* Extract the slice from the search request so it'll be available in the status. This is potentially useful for users that manually - * slice their search requests so they can keep track of it and **absolutely** useful for automatically sliced reindex requests so - * they can properly track the responses. *//* - Integer sliceId = searchRequest.source().slice() == null ? null : searchRequest.source().slice().getId(); - return new WorkingBulkByScrollTask(id, type, action, getDescription(), parentTaskId, sliceId, requestsPerSecond);*/ - return new BulkByScrollTask(id, type, action, getDescription(), parentTaskId); } @@ -408,9 +399,9 @@ public void readFrom(StreamInput in) throws IOException { maxRetries = in.readVInt(); requestsPerSecond = in.readFloat(); if (in.getVersion().onOrAfter(Version.V_5_1_1)) { - slices = new SlicesCount(in); + slices = new Slices(in); } else { - slices = SlicesCount.DEFAULT; + slices = Slices.DEFAULT; } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java index dc7a394b58bc6..49317714ef310 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java @@ -145,7 +145,7 @@ public Self setShouldStoreResult(boolean shouldStoreResult) { /** * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. */ - public Self setSlices(SlicesCount slices) { + public Self setSlices(Slices slices) { request.setSlices(slices); return self(); } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 160129f631b5e..2a1f3c35e3fd0 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -31,7 +31,6 @@ import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; import java.util.Collections; @@ -40,11 +39,15 @@ import static java.lang.Math.min; import static java.util.Collections.emptyList; -import static java.util.Collections.unmodifiableList; import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; /** * Task storing information about a currently running BulkByScroll request. + * + * When the request is not sliced, this task is the only task created, and starts an action to perform search requests. + * + * When the request is sliced, this task can either represent a parent coordinating task (using {@link BulkByScrollTask#setParent(int)}) or + * a child task that performs search queries (using {@link BulkByScrollTask#setChild(Integer, float)}). */ public class BulkByScrollTask extends CancellableTask { @@ -72,10 +75,16 @@ private BulkByScrollTask.Status emptyStatus() { return new Status(Collections.emptyList(), getReasonCancelled()); } + /** + * Returns true if this task is a parent of other sliced tasks. False otherwise + */ public boolean isParent() { return parentWorker != null; } + /** + * Sets this task to be a parent task for {@code slices} sliced subtasks + */ public void setParent(int slices) { if (isParent()) { throw new IllegalStateException("Parent worker is already set"); @@ -87,14 +96,29 @@ public void setParent(int slices) { parentWorker = new ParentBulkByScrollWorker(this, slices); } + /** + * Returns the worker object that tracks the state of sliced subtasks. Throws IllegalStateException if this task is not set to be + * a parent task. + */ public ParentBulkByScrollWorker getParentWorker() { + if (!isParent()) { + throw new IllegalStateException("This task is not set as a parent worker"); + } return parentWorker; } + /** + * Returns true if this task is a child task that performs search requests. False otherwise + */ public boolean isChild() { return childWorker != null; } + /** + * Sets this task to be a child task that performs search requests + * @param sliceId If this is is a sliced task, which slice number this task corresponds to + * @param requestsPerSecond How many search requests per second this task should make + */ public void setChild(Integer sliceId, float requestsPerSecond) { if (isChild()) { throw new IllegalStateException("Child worker is already set"); @@ -106,7 +130,14 @@ public void setChild(Integer sliceId, float requestsPerSecond) { childWorker = new ChildBulkByScrollWorker(this, sliceId, requestsPerSecond); } + /** + * Returns the worker object that manages sending search requests. Throws IllegalStateException if this task is not set to be a + * child task. + */ public ChildBulkByScrollWorker getChildWorker() { + if (!isChild()) { + throw new IllegalStateException("This task is not set as a child worker"); + } return childWorker; } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index 6e7a52fcaccb0..0b21b152d2bc5 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -199,8 +199,10 @@ private void setRequestsPerSecond(float requestsPerSecond) { this.requestsPerSecond = requestsPerSecond; } + /** + * Apply {@code newRequestsPerSecond} as the new rate limit for this task's search requests + */ public void rethrottle(float newRequestsPerSecond) { - logger.warn("[{}]: STARTING CHILD RETHROTTLE", task.getId()); synchronized (delayedPrepareBulkRequestReference) { logger.debug("[{}]: rethrottling to [{}] requests per second", task.getId(), newRequestsPerSecond); setRequestsPerSecond(newRequestsPerSecond); @@ -214,7 +216,6 @@ public void rethrottle(float newRequestsPerSecond) { this.delayedPrepareBulkRequestReference.set(delayedPrepareBulkRequest.rethrottle(newRequestsPerSecond)); } - logger.warn("[{}]: FINISH CHILD RETHROTTLE HANDLING TASK", task.getId()); } class DelayedPrepareBulkRequest { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java index f9ed1583d12fe..0437a525b6d0f 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java @@ -30,6 +30,9 @@ import static java.util.Collections.unmodifiableList; +/** + * Tracks the state of sliced subtasks and provides unified status information for a sliced BulkByScrollRequest. + */ public class ParentBulkByScrollWorker { private final BulkByScrollTask task; @@ -40,17 +43,20 @@ public class ParentBulkByScrollWorker { */ private final AtomicArray results; /** - * How many subtasks are still running TODO rename this + * How many subtasks are still running */ - private final AtomicInteger counter; + private final AtomicInteger runningSubtasks; public ParentBulkByScrollWorker(BulkByScrollTask task, int slices) { this.task = task; this.slices = slices; results = new AtomicArray<>(slices); - counter = new AtomicInteger(slices); + runningSubtasks = new AtomicInteger(slices); } + /** + * Returns the number of slices this BulkByScrollRequest will use + */ public int getSlices() { return slices; } @@ -62,10 +68,16 @@ public BulkByScrollTask.Status getStatus() { return new BulkByScrollTask.Status(unmodifiableList(statuses), task.getReasonCancelled()); } + /** + * The number of sliced subtasks that are still running + */ public int runningSliceSubTasks() { - return counter.get(); + return runningSubtasks.get(); } + /** + * Build the status for this task given a snapshot of the information of running slices. + */ public TaskInfo getStatusGivenSlicesTaskInfo(String localNodeId, List sliceInfo) { /* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the * finished status because we don't expect them to change after the task is finished. */ @@ -109,7 +121,7 @@ public void onSliceFailure(ActionListener listener, int sl } private void recordSliceCompletionAndRespondIfAllDone(ActionListener listener) { - if (counter.decrementAndGet() != 0) { + if (runningSubtasks.decrementAndGet() != 0) { return; } List responses = new ArrayList<>(results.length()); diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java b/core/src/main/java/org/elasticsearch/index/reindex/Slices.java similarity index 73% rename from core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java rename to core/src/main/java/org/elasticsearch/index/reindex/Slices.java index bdffac8e2fc72..0c845d5c790e5 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SlicesCount.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/Slices.java @@ -28,35 +28,45 @@ import java.io.IOException; import java.util.Objects; -public final class SlicesCount implements ToXContent, Writeable { +/** + * Represents the setting for the number of slices used by a BulkByScrollRequest. Valid values are positive integers and "auto". The + * "auto" setting defers choosing the number of slices to the request handler. + */ +public final class Slices implements ToXContent, Writeable { private static final int AUTO_COUNT = -1; public static final String FIELD_NAME = "slices"; public static final String AUTO_VALUE = "auto"; - public static final SlicesCount AUTO = new SlicesCount(AUTO_COUNT); - public static final SlicesCount ONE = of(1); - public static final SlicesCount DEFAULT = ONE; + public static final Slices AUTO = new Slices(AUTO_COUNT); + public static final Slices ONE = of(1); + public static final Slices DEFAULT = ONE; private final int count; - private SlicesCount(int count) { + private Slices(int count) { this.count = count; } - public SlicesCount(StreamInput stream) throws IOException { + public Slices(StreamInput stream) throws IOException { count = stream.readVInt(); } - public static SlicesCount of(int count) { + /** + * Creates a new {@link Slices} as a concrete number. The given number must be a positive integer. + */ + public static Slices of(int count) { if (count < 1) { throw new IllegalArgumentException("[slices] must be at least 1"); } - return new SlicesCount(count); + return new Slices(count); } - public static SlicesCount parse(String slicesString) { + /** + * Parse a string to a valid {@link Slices} value + */ + public static Slices parse(String slicesString) { Objects.requireNonNull(slicesString); if (AUTO_VALUE.equals(slicesString)) { @@ -71,14 +81,23 @@ public static SlicesCount parse(String slicesString) { } } + /** + * Returns true if this value is set as "auto", false otherwise + */ public boolean isAuto() { return count == AUTO_COUNT; } + /** + * Returns true if this value is set as a number, false otherwise + */ public boolean isNumber() { return !isAuto(); } + /** + * Returns the number value this {@link Slices} has, if it is set as a number. Otherwise throws IllegalStateException + */ public int number() { if (isAuto()) { throw new IllegalStateException("Slice count is set as \"auto\", not a number"); @@ -113,7 +132,7 @@ public boolean equals(Object obj) { return false; } - SlicesCount other = (SlicesCount) obj; + Slices other = (Slices) obj; return other.count == count; } diff --git a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java index 645b5be418dee..49e84683906dc 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java @@ -47,9 +47,13 @@ public void testForSlice() { int actualSlices = between(2, 1000); if (randomBoolean()) { - original.setSlices(SlicesCount.of(actualSlices)); + original.setSlices(Slices.of(actualSlices)); } else { - original.setSlices(SlicesCount.AUTO); + /* + * In this case we pretend that the true number of slices will be randomSlices, since we're just verifying the request + * behavior and not the slicing itself + */ + original.setSlices(Slices.AUTO); } TaskId slicingTask = new TaskId(randomAlphaOfLength(5), randomLong()); @@ -62,7 +66,7 @@ public void testForSlice() { assertEquals(original.getRetryBackoffInitialTime(), forSliced.getRetryBackoffInitialTime()); assertEquals(original.getMaxRetries(), forSliced.getMaxRetries()); assertEquals("only the parent task should store results", false, forSliced.getShouldStoreResult()); - assertEquals("slice requests always have a single worker", SlicesCount.ONE, forSliced.getSlices()); + assertEquals("slice requests always have a single worker", Slices.ONE, forSliced.getSlices()); assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / actualSlices, forSliced.getRequestsPerSecond(), Float.MIN_NORMAL); assertEquals("size is split evenly between all workers", original.getSize() == AbstractBulkByScrollRequest.SIZE_ALL_MATCHES diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index fc67267f74e67..ad0715c6584f3 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -50,7 +50,7 @@ public void testReindexFromRemoteDoesNotSupportWorkers() { reindex.setRemoteInfo( new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); - reindex.setSlices(SlicesCount.of(between(2, Integer.MAX_VALUE))); + reindex.setSlices(Slices.of(between(2, Integer.MAX_VALUE))); ActionRequestValidationException e = reindex.validate(); assertEquals( "Validation Failed: 1: reindex from remote sources doesn't support workers > 1 but was [" + reindex.getSlices() + "];", @@ -60,7 +60,7 @@ public void testReindexFromRemoteDoesNotSupportWorkers() { public void testNoSliceWithWorkers() { ReindexRequest reindex = newRequest(); reindex.getSearchRequest().source().slice(new SliceBuilder(0, 4)); - reindex.setSlices(SlicesCount.of(between(2, Integer.MAX_VALUE))); + reindex.setSlices(Slices.of(between(2, Integer.MAX_VALUE))); ActionRequestValidationException e = reindex.validate(); assertEquals("Validation Failed: 1: can't specify both slice and workers;", e.getMessage()); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java index d12508ff065f1..22bad846e3037 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -91,9 +91,9 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) { request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh())); request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout())); - String slices = restRequest.param(SlicesCount.FIELD_NAME); + String slices = restRequest.param(Slices.FIELD_NAME); if (slices != null) { - request.setSlices(SlicesCount.parse(slices)); + request.setSlices(Slices.parse(slices)); } String waitForActiveShards = restRequest.param("wait_for_active_shards"); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 93f0e26655980..2b592701d3f1b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -48,10 +48,15 @@ class BulkByScrollParallelizationHelper { private BulkByScrollParallelizationHelper() {} + /** + * Figures out how many slices to use when handling this request. If the {@code request} has slices set as a number, that number + * will be used. If set to {@code "auto"}, it will compute it from the source indices' properties. The given consumer is passed the + * resulting number of slices in either case. + */ public static > void computeSlicing(Client client, Request request, ActionListener listener, Consumer slicedBehavior) { - SlicesCount slices = request.getSlices(); + Slices slices = request.getSlices(); if (slices.isAuto()) { client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( response -> slicedBehavior.accept(sliceBasedOnShards(response)), @@ -80,14 +85,15 @@ public static > void startS BulkByScrollTask task, Request request, ActionListener listener) { - int totalSlices = task.getParentWorker().getSlices(); + ParentBulkByScrollWorker worker = task.getParentWorker(); + int totalSlices = worker.getSlices(); TaskId parentTaskId = new TaskId(localNodeId, task.getId()); for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, totalSlices)) { // TODO move the request to the correct node. maybe here or somehow do it as part of startup for reindex in general.... Request requestForSlice = request.forSlice(parentTaskId, slice, totalSlices); ActionListener sliceListener = ActionListener.wrap( - r -> task.getParentWorker().onSliceResponse(listener, slice.source().slice().getId(), r), - e -> task.getParentWorker().onSliceFailure(listener, slice.source().slice().getId(), e)); + r -> worker.onSliceResponse(listener, slice.source().slice().getId(), r), + e -> worker.onSliceFailure(listener, slice.source().slice().getId(), e)); client.execute(action, requestForSlice, sliceListener); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 1c8a5682113fb..d0704aa65826b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -52,7 +52,6 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { - logger.error("SLICES {} TASK {}", slices, task.getClass()); BulkByScrollTask bulkTask = (BulkByScrollTask) task; @@ -69,19 +68,6 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, - clusterService.localNode().getId(), parentTask, request, slices, listener); - } else { - ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, - listener).start(); - }*/ }); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index a289796c99ae8..85118be2b3d9c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -80,9 +80,7 @@ import java.util.function.Function; import static java.util.Collections.emptyList; -import static java.util.Collections.list; import static java.util.Collections.synchronizedList; -import static java.util.Objects.compare; import static java.util.Objects.requireNonNull; import static org.elasticsearch.index.VersionType.INTERNAL; @@ -132,21 +130,6 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, - clusterService.localNode().getId(), parentTask, request, slices, listener); - } else { - checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); - ClusterState state = clusterService.state(); - validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), - indexNameExpressionResolver, autoCreateIndex, state); - ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), task); - new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, assigningClient, threadPool, request, scriptService, - state, listener).start(); - }*/ }); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index 6b2c976c0fe80..8722cb8d8b993 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -54,9 +54,7 @@ public TransportRethrottleAction(Settings settings, ThreadPool threadPool, Clust @Override protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, ActionListener listener) { - logger.warn("[{}]: RETHROTTLE start taskOperation", task.getId()); rethrottle(logger, clusterService.localNode().getId(), client, task, request.getRequestsPerSecond(), listener); - logger.warn("[{}]: RETHROTTLE finish taskOperation", task.getId()); } static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, @@ -64,17 +62,14 @@ static void rethrottle(Logger logger, String localNodeId, Client client, BulkByS if (task.isChild()) { rethrottleChildTask(logger, localNodeId, task, newRequestsPerSecond, listener); - logger.warn("[{}]: FINISH CHILD INIT RETHROTTLE", task.getId()); return; } if (task.isParent()) { rethrottleParentTask(logger, localNodeId, client, task, newRequestsPerSecond, listener); - logger.warn("[{}]: FINISH PARENT INIT RETHROTTLE", task.getId()); return; } - logger.warn("[{}]: THROW EXCEPTION RETHROTTLE", task.getId()); throw new IllegalArgumentException("task [" + task.getId() + "] must be set as a child or parent"); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 77d3c5a6c1796..a659b10dc4b43 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -81,19 +81,6 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); new AsyncIndexBySearchAction(bulkTask, logger, client, threadPool, request, scriptService, state, listener).start(); } - - -/* if (slices > 1) { - ParentBulkByScrollTask parentTask = (ParentBulkByScrollTask) task; - parentTask.setSlices(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, - clusterService.localNode().getId(), parentTask, request, slices, listener); - } else { - ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, - listener).start(); - }*/ }); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 78f734d036548..7a6c8358c893b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -143,7 +143,7 @@ public void setupForTest() { scrollId = null; taskManager = new TaskManager(Settings.EMPTY); testTask = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest); - testTask.setChild(null, testRequest.getRequestsPerSecond()); // todo this is awkward - is there a way to set this stuff in the task or somewhere else + testTask.setChild(null, testRequest.getRequestsPerSecond()); worker = testTask.getChildWorker(); localNode = new DiscoveryNode("thenode", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -590,7 +590,7 @@ public void testCancelBeforeRefreshAndFinish() throws Exception { assertNull("No refresh was attempted", client.lastRefreshRequest.get()); } - /* + /** * Tests that we can cancel the request during its throttling delay. This can't use {@link #cancelTaskCase(Consumer)} because it needs * to send the request un-canceled and cancel it at a specific time. */ diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index 5a378d4286f13..726e6898e08f4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -232,7 +232,7 @@ public void testDeleteByQueryCancel() throws Exception { public void testReindexCancelWithWorkers() throws Exception { testCancel(ReindexAction.NAME, - reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(SlicesCount.of(5)), + reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(Slices.of(5)), (response, total, modified) -> { assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); refresh("dest"); @@ -250,7 +250,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { "}"); assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); - testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(SlicesCount.of(5)), + testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(Slices.of(5)), (response, total, modified) -> { assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified); @@ -260,7 +260,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { } public void testDeleteByQueryCancelWithWorkers() throws Exception { - testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(SlicesCount.of(5)), + testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(Slices.of(5)), (response, total, modified) -> { assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index e3e69a038ab44..f63c5fb3a2d64 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -235,16 +235,14 @@ public void testSlices() throws Exception { ); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); - logger.error("SHARDSCOUNT {}", getNumShards("test").numPrimaries); - // Deletes the two docs that matches "foo:a" - assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.of(5)).get(), + assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(Slices.of(5)).get(), matcher().deleted(2).slices(hasSize(5))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); // Delete remaining docs DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) - .setSlices(SlicesCount.of(5)); + .setSlices(Slices.of(5)); assertThat(request.get(), matcher().deleted(5).slices(hasSize(5))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } @@ -263,7 +261,7 @@ public void testSlicesAuto() throws Exception { NumShards numShards = getNumShards("test"); int numSlicesExpected = numShards.numPrimaries > 1 - ? numShards.numPrimaries + ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) : 0; // Deletes the two docs that matches "foo:a" @@ -272,7 +270,7 @@ public void testSlicesAuto() throws Exception { .source("test") .filter(termQuery("foo", "a")) .refresh(true) - .setSlices(SlicesCount.AUTO).get(), + .setSlices(Slices.AUTO).get(), matcher() .deleted(2) .slices(hasSize(numSlicesExpected))); @@ -283,7 +281,7 @@ public void testSlicesAuto() throws Exception { .source("test") .filter(QueryBuilders.matchAllQuery()) .refresh(true) - .setSlices(SlicesCount.AUTO).get(), + .setSlices(Slices.AUTO).get(), matcher() .deleted(5) .slices(hasSize(numSlicesExpected))); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index d179316a1b8ec..49774795ca84e 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -100,7 +100,7 @@ public void testCopyManyWithSlices() throws Exception { assertHitCount(client().prepareSearch("source").setSize(0).get(), max); // Copy all the docs - ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(SlicesCount.of(workers)); + ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(Slices.of(workers)); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(workers))); @@ -108,7 +108,7 @@ public void testCopyManyWithSlices() throws Exception { // Copy some of the docs int half = max / 2; - copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(SlicesCount.of(workers)); + copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(Slices.of(workers)); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); copy.size(half); // The real "size" of the request. @@ -129,16 +129,16 @@ public void testCopyManyWithSlicesAuto() throws Exception { NumShards numShards = getNumShards("source"); int numSlicesExpected = numShards.numPrimaries > 1 - ? numShards.numPrimaries + ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) : 0; - ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(SlicesCount.AUTO); + ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(Slices.AUTO); copy.source().setSize(5); assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(numSlicesExpected))); assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max); int half = max / 2; - copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(SlicesCount.AUTO); + copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(Slices.AUTO); copy.source().setSize(5); copy.size(half); BulkByScrollResponse response = copy.get(); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index cbe1758457af0..33c57804be7ff 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -60,15 +60,23 @@ public void testDeleteByQuery() throws Exception { } public void testReindexWithWorkers() throws Exception { - testCase(reindex().source("test").destination("dest").setSlices(SlicesCount.of(between(2, 10))), ReindexAction.NAME); + testCase(reindex().source("test").destination("dest").setSlices(randomSlices()), ReindexAction.NAME); } public void testUpdateByQueryWithWorkers() throws Exception { - testCase(updateByQuery().source("test").setSlices(SlicesCount.of(between(2, 10))), UpdateByQueryAction.NAME); + testCase(updateByQuery().source("test").setSlices(randomSlices()), UpdateByQueryAction.NAME); } public void testDeleteByQueryWithWorkers() throws Exception { - testCase(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).setSlices(SlicesCount.of(between(2, 10))), DeleteByQueryAction.NAME); + testCase(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).setSlices(randomSlices()), DeleteByQueryAction.NAME); + } + + private static Slices randomSlices() { + if (randomBoolean()) { + return Slices.AUTO; + } else { + return Slices.of(between(2, 10)); + } } private void testCase(AbstractBulkByScrollRequestBuilder request, String actionName) throws Exception { @@ -76,32 +84,36 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a /* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches. * we can't rely on all of them doing so, but */ + + createIndex("test"); + int numSlices; + if (request.request().getSlices().isAuto()) { + NumShards numShards = getNumShards("test"); + numSlices = Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); + } else { + numSlices = request.request().getSlices().number(); + } + List docs = new ArrayList<>(); - for (int i = 0; i < request.request().getSlices().number() * 10; i++) { // todo need to be aware for auto + for (int i = 0; i < numSlices * 10; i++) { docs.add(client().prepareIndex("test", "test", Integer.toString(i)).setSource("foo", "bar")); } indexRandom(true, docs); - logger.info("Executing request"); - // Start a request that will never finish unless we rethrottle it request.setRequestsPerSecond(.000001f); // Throttle "forever" request.source().setSize(1); // Make sure we use multiple batches ActionFuture responseListener = request.execute(); - logger.info("Finished executing request"); - - TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices().number()); + TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, numSlices); TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId(); - if (request.request().getSlices().number() == 1) { - logger.info("There are no child tasks"); + if (numSlices == 1) { assertThat(taskGroupToRethrottle.getChildTasks(), empty()); } else { - logger.info("Asserting child tasks are running"); // There should be a sane number of child tasks running assertThat(taskGroupToRethrottle.getChildTasks(), - hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(request.request().getSlices().number())))); + hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(numSlices)))); // Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do) assertBusy(() -> { BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get() @@ -109,13 +121,11 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a long finishedSubTasks = parent.getSliceStatuses().stream().filter(Objects::nonNull).count(); ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get(); list.rethrowFailures("subtasks"); - assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) request.request().getSlices().number())); + assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) numSlices)); assertThat(list.getTasks().size(), greaterThan(0)); }); } - logger.info("Rethrottling task [{}]", taskToRethrottle.getId()); - // Now rethrottle it so it'll finish float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000; // No throttle or "very fast" ListTasksResponse rethrottleResponse = rethrottle().setTaskId(taskToRethrottle).setRequestsPerSecond(newRequestsPerSecond).get(); @@ -123,10 +133,8 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a assertThat(rethrottleResponse.getTasks(), hasSize(1)); BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus(); - logger.info("Finished executing rethrottle task"); - // Now check the resulting requests per second. - if (request.request().getSlices().number() == 1) { + if (numSlices == 1) { // If there is a single slice it should match perfectly assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL); } else { @@ -139,7 +147,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F; float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? - Float.POSITIVE_INFINITY : (newRequestsPerSecond / request.request().getSlices().number()) * 0.99F; + Float.POSITIVE_INFINITY : (newRequestsPerSecond / numSlices) * 0.99F; boolean oneSliceRethrottled = false; float totalRequestsPerSecond = 0; for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) { @@ -172,14 +180,10 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f); } - logger.info("Asserted a bunch of stuff"); - // Now the response should come back quickly because we've rethrottled the request BulkByScrollResponse response = responseListener.get(); assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", - response.getBatches(), greaterThanOrEqualTo(request.request().getSlices().number())); - - logger.info("Test case is done"); + response.getBatches(), greaterThanOrEqualTo(numSlices)); } private TaskGroup findTaskToRethrottle(String actionName, int sliceCount) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index 31767034d897a..d97f57aab489b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -71,14 +71,14 @@ public void testReindexRequest() throws IOException { assertRequestEquals(reindex, tripped); // Try slices with a version that doesn't support slices. That should fail. - reindex.setSlices(SlicesCount.of(between(2, 1000))); + reindex.setSlices(Slices.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, reindex, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new ReindexRequest(); - reindex.setSlices(SlicesCount.of(1)); + reindex.setSlices(Slices.of(1)); roundTrip(Version.V_5_0_0_rc1, reindex, tripped); assertRequestEquals(Version.V_5_0_0_rc1, reindex, tripped); } @@ -95,14 +95,14 @@ public void testUpdateByQueryRequest() throws IOException { assertEquals(update.getPipeline(), tripped.getPipeline()); // Try slices with a version that doesn't support slices. That should fail. - update.setSlices(SlicesCount.of(between(2, 1000))); + update.setSlices(Slices.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, update, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new UpdateByQueryRequest(); - update.setSlices(SlicesCount.of(1)); + update.setSlices(Slices.of(1)); roundTrip(Version.V_5_0_0_rc1, update, tripped); assertRequestEquals(update, tripped); assertEquals(update.getPipeline(), tripped.getPipeline()); @@ -116,14 +116,14 @@ public void testDeleteByQueryRequest() throws IOException { assertRequestEquals(delete, tripped); // Try slices with a version that doesn't support slices. That should fail. - delete.setSlices(SlicesCount.of(between(2, 1000))); + delete.setSlices(Slices.of(between(2, 1000))); Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, delete, null)); assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); // Try without slices with a version that doesn't support slices. That should work. tripped = new DeleteByQueryRequest(); - delete.setSlices(SlicesCount.of(1)); + delete.setSlices(Slices.of(1)); roundTrip(Version.V_5_0_0_rc1, delete, tripped); assertRequestEquals(delete, tripped); } @@ -139,7 +139,11 @@ private void randomRequest(AbstractBulkByScrollRequest request) { request.setTimeout(TimeValue.parseTimeValue(randomTimeValue(), null, "test")); request.setWaitForActiveShards(randomIntBetween(0, 10)); request.setRequestsPerSecond(between(0, Integer.MAX_VALUE)); - request.setSlices(SlicesCount.of(between(1, Integer.MAX_VALUE))); + + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(0, Integer.MAX_VALUE)); + request.setSlices(slices); } private void randomRequest(AbstractBulkIndexByScrollRequest request) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index 8e9112898635e..e0af18a202a1b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -73,18 +73,18 @@ public void testSlices() throws Exception { assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); // Reindex all the docs - assertThat(updateByQuery().source("test").refresh(true).setSlices(SlicesCount.of(5)).get(), matcher().updated(4).slices(hasSize(5))); + assertThat(updateByQuery().source("test").refresh(true).setSlices(Slices.of(5)).get(), matcher().updated(4).slices(hasSize(5))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now none of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(SlicesCount.of(5)).refresh(true).get(), + assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(Slices.of(5)).refresh(true).get(), matcher().updated(0).slices(hasSize(5))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); // Now half of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(SlicesCount.of(5)).get(), + assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(Slices.of(5)).get(), matcher().updated(2).slices(hasSize(5))); assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); @@ -103,7 +103,7 @@ public void testSlicesAuto() throws Exception { NumShards numShards = getNumShards("test"); int numSlicesExpected = numShards.numPrimaries > 1 - ? numShards.numPrimaries + ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) : 0; // Reindex all the docs @@ -111,7 +111,7 @@ public void testSlicesAuto() throws Exception { updateByQuery() .source("test") .refresh(true) - .setSlices(SlicesCount.AUTO).get(), + .setSlices(Slices.AUTO).get(), matcher() .updated(4) .slices(hasSize(numSlicesExpected))); @@ -123,7 +123,7 @@ public void testSlicesAuto() throws Exception { updateByQuery() .source("test") .filter(termQuery("foo", "no_match")) - .setSlices(SlicesCount.AUTO) + .setSlices(Slices.AUTO) .refresh(true).get(), matcher() .updated(0) @@ -137,7 +137,7 @@ public void testSlicesAuto() throws Exception { .source("test") .filter(termQuery("foo", "a")) .refresh(true) - .setSlices(SlicesCount.AUTO).get(), + .setSlices(Slices.AUTO).get(), matcher() .updated(2) .slices(hasSize(numSlicesExpected))); From 679c2a0a5cfe6b6f11a6a7f710b8806f58be12d9 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 1 Aug 2017 09:31:42 -0700 Subject: [PATCH 11/21] wip #24547 checkstyle fixes --- .../index/reindex/ChildBulkByScrollWorker.java | 18 +++++++++++++++--- .../AbstractAsyncBulkByScrollAction.java | 2 +- .../index/reindex/TransportReindexAction.java | 10 ++++++---- .../reindex/TransportRethrottleAction.java | 6 ++++-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index 0b21b152d2bc5..52f30bd6a12b9 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -81,9 +81,21 @@ public ChildBulkByScrollWorker(BulkByScrollTask task, Integer sliceId, float req } public BulkByScrollTask.Status getStatus() { - return new BulkByScrollTask.Status(sliceId, total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), - noops.get(), bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), - task.getReasonCancelled(), throttledUntil()); + return new BulkByScrollTask.Status( + sliceId, + total.get(), + updated.get(), + created.get(), + deleted.get(), + batch.get(), + versionConflicts.get(), + noops.get(), + bulkRetries.get(), + searchRetries.get(), + timeValueNanos(throttledNanos.get()), + getRequestsPerSecond(), + task.getReasonCancelled(), + throttledUntil()); } public void handleCancel() { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 2da9517941958..24321ef68a609 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -766,7 +766,7 @@ public static RequestWrapper wrap(DeleteRequest request) { /** * Apply a {@link Script} to a {@link RequestWrapper} */ - public static abstract class ScriptApplier implements BiFunction, ScrollableHitSource.Hit, RequestWrapper> { + public abstract static class ScriptApplier implements BiFunction, ScrollableHitSource.Hit, RequestWrapper> { private final ChildBulkByScrollWorker taskWorker; private final ScriptService scriptService; diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 9b12bc477ee78..16b3d7e19982b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -128,7 +128,8 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener()); RestClient restClient = buildRestClient(remoteInfo, task.getId(), createdThreads); - return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, restClient, - remoteInfo.getQuery(), mainRequest.getSearchRequest()); + return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, worker::countSearchRetry, this::finishHim, + restClient, remoteInfo.getQuery(), mainRequest.getSearchRequest()); } return super.buildScrollableResultSource(backoffPolicy); } @@ -395,7 +396,8 @@ protected void copyRouting(RequestWrapper request, String routing) { class ReindexScriptApplier extends ScriptApplier { - public ReindexScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, Map params) { + ReindexScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, + Map params) { super(taskWorker, scriptService, script, params); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index 8722cb8d8b993..e1ee629d79894 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -73,7 +73,8 @@ static void rethrottle(Logger logger, String localNodeId, Client client, BulkByS throw new IllegalArgumentException("task [" + task.getId() + "] must be set as a child or parent"); } - private static void rethrottleParentTask(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { + private static void rethrottleParentTask(Logger logger, String localNodeId, Client client, BulkByScrollTask task, + float newRequestsPerSecond, ActionListener listener) { final ParentBulkByScrollWorker parentWorker = task.getParentWorker(); final int runningSubtasks = parentWorker.runningSliceSubTasks(); @@ -95,7 +96,8 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie } } - private static void rethrottleChildTask(Logger logger, String localNodeId, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { + private static void rethrottleChildTask(Logger logger, String localNodeId, BulkByScrollTask task, float newRequestsPerSecond, + ActionListener listener) { logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond); task.getChildWorker().rethrottle(newRequestsPerSecond); listener.onResponse(task.taskInfo(localNodeId, true)); From a0a30faefcad9457acde00aada79ac4c09f99d0a Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 1 Aug 2017 14:53:55 -0700 Subject: [PATCH 12/21] wip #24547 consolidate not auto slicing and auto slicing unit tests add unit tests with multiple sources --- .../reindex/DeleteByQueryBasicTests.java | 88 ++++++++++------- .../index/reindex/ReindexBasicTests.java | 76 +++++++++------ .../index/reindex/ReindexTestCase.java | 25 +++++ .../reindex/UpdateByQueryBasicTests.java | 96 +++++++++++-------- 4 files changed, 182 insertions(+), 103 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index f63c5fb3a2d64..eae19a4088b3c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -30,7 +30,10 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY_ALLOW_DELETE; @@ -235,34 +238,10 @@ public void testSlices() throws Exception { ); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); - // Deletes the two docs that matches "foo:a" - assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(Slices.of(5)).get(), - matcher().deleted(2).slices(hasSize(5))); - assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); - - // Delete remaining docs - DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) - .setSlices(Slices.of(5)); - assertThat(request.get(), matcher().deleted(5).slices(hasSize(5))); - assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); - } - - public void testSlicesAuto() throws Exception { - indexRandom(true, - client().prepareIndex("test", "test", "1").setSource("foo", "a"), - client().prepareIndex("test", "test", "2").setSource("foo", "a"), - client().prepareIndex("test", "test", "3").setSource("foo", "b"), - client().prepareIndex("test", "test", "4").setSource("foo", "c"), - client().prepareIndex("test", "test", "5").setSource("foo", "d"), - client().prepareIndex("test", "test", "6").setSource("foo", "e"), - client().prepareIndex("test", "test", "7").setSource("foo", "f") - ); - assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); - - NumShards numShards = getNumShards("test"); - int numSlicesExpected = numShards.numPrimaries > 1 - ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) - : 0; + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(2, 10)); + int expectedSlices = expectedSlices(slices, "test"); // Deletes the two docs that matches "foo:a" assertThat( @@ -270,24 +249,67 @@ public void testSlicesAuto() throws Exception { .source("test") .filter(termQuery("foo", "a")) .refresh(true) - .setSlices(Slices.AUTO).get(), + .setSlices(slices).get(), matcher() .deleted(2) - .slices(hasSize(numSlicesExpected))); + .slices(hasSize(expectedSlices))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5); // Delete remaining docs - assertThat(deleteByQuery() + assertThat( + deleteByQuery() .source("test") .filter(QueryBuilders.matchAllQuery()) .refresh(true) - .setSlices(Slices.AUTO).get(), + .setSlices(slices).get(), matcher() .deleted(5) - .slices(hasSize(numSlicesExpected))); + .slices(hasSize(expectedSlices))); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0); } + public void testMultipleSources() throws Exception { + int sourceIndices = between(2, 5); + + Map> docs = new HashMap<>(); + for (int sourceIndex = 0; sourceIndex < sourceIndices; sourceIndex++) { + String indexName = "test" + sourceIndex; + docs.put(indexName, new ArrayList<>()); + int numDocs = between(5, 15); + for (int i = 0; i < numDocs; i++) { + docs.get(indexName).add(client().prepareIndex(indexName, "test", Integer.toString(i)).setSource("foo", "a")); + } + } + + List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + indexRandom(true, allDocs); + for (Map.Entry> entry : docs.entrySet()) { + assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); + } + + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(1, 10)); + int expectedSlices = expectedSlices(slices, docs.keySet()); + + String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); + + assertThat( + deleteByQuery() + .source(sourceIndexNames) + .filter(QueryBuilders.matchAllQuery()) + .refresh(true) + .setSlices(slices).get(), + matcher() + .deleted(allDocs.size()) + .slices(hasSize(expectedSlices))); + + for (String index : docs.keySet()) { + assertHitCount(client().prepareSearch(index).setTypes("test").setSize(0).get(), 0); + } + + } + /** * Test delete by query support for filtering by type. This entire feature * can and should be removed when we drop support for types index with diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index 49774795ca84e..c8bccb8bea17c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -22,7 +22,11 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -88,8 +92,6 @@ public void testCopyMany() throws Exception { } public void testCopyManyWithSlices() throws Exception { - int workers = between(2, 10); - List docs = new ArrayList<>(); int max = between(150, 500); for (int i = 0; i < max; i++) { @@ -99,51 +101,65 @@ public void testCopyManyWithSlices() throws Exception { indexRandom(true, docs); assertHitCount(client().prepareSearch("source").setSize(0).get(), max); + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(2, 10)); + int expectedSlices = expectedSlices(slices, "source"); + // Copy all the docs - ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(Slices.of(workers)); + ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(slices); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); - assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(workers))); + assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(expectedSlices))); assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max); // Copy some of the docs int half = max / 2; - copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(Slices.of(workers)); + copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(slices); // Use a small batch size so we have to use more than one batch copy.source().setSize(5); copy.size(half); // The real "size" of the request. BulkByScrollResponse response = copy.get(); - assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(workers))); + assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(expectedSlices))); assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated()); } - public void testCopyManyWithSlicesAuto() throws Exception { - int max = between(150, 500); - List docs = new ArrayList<>(); - for (int i = 0; i < max; i++) { - docs.add(client().prepareIndex("source", "test", Integer.toString(i)).setSource("foo", "a")); + public void testMultipleSources() throws Exception { + int sourceIndices = between(2, 5); + + Map> docs = new HashMap<>(); + for (int sourceIndex = 0; sourceIndex < sourceIndices; sourceIndex++) { + String indexName = "source" + sourceIndex; + String typeName = "test" + sourceIndex; + docs.put(indexName, new ArrayList<>()); + int numDocs = between(50, 200); + for (int i = 0; i < numDocs; i++) { + docs.get(indexName).add(client().prepareIndex(indexName, typeName, "id_" + sourceIndex + "_" + i).setSource("foo", "a")); + } } - indexRandom(true, docs); - assertHitCount(client().prepareSearch("source").setSize(0).get(), max); - - NumShards numShards = getNumShards("source"); - int numSlicesExpected = numShards.numPrimaries > 1 - ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) - : 0; - - ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(Slices.AUTO); - copy.source().setSize(5); - assertThat(copy.get(), matcher().created(max).batches(greaterThanOrEqualTo(max / 5)).slices(hasSize(numSlicesExpected))); - assertHitCount(client().prepareSearch("dest").setTypes("type").setSize(0).get(), max); + List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + indexRandom(true, allDocs); + for (Map.Entry> entry : docs.entrySet()) { + assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); + } - int half = max / 2; - copy = reindex().source("source").destination("dest_half", "type").refresh(true).setSlices(Slices.AUTO); - copy.source().setSize(5); - copy.size(half); - BulkByScrollResponse response = copy.get(); - assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(numSlicesExpected))); - assertHitCount(client().prepareSearch("dest_half").setSize(0).get(), response.getCreated()); + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(1, 10)); + int expectedSlices = expectedSlices(slices, docs.keySet()); + + String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); + ReindexRequestBuilder request = reindex() + .source(sourceIndexNames) + .destination("dest", "type") + .refresh(true) + .setSlices(slices); + + BulkByScrollResponse response = request.get(); + assertThat(response, matcher().created(allDocs.size()).slices(hasSize(expectedSlices))); + assertHitCount(client().prepareSearch("dest").setSize(0).get(), allDocs.size()); } + } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java index fcf80ea283cf4..873c24e68d7af 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java @@ -25,7 +25,10 @@ import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.stream.Collectors; +import static java.util.Collections.singleton; import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; /** @@ -62,4 +65,26 @@ protected RethrottleRequestBuilder rethrottle() { public static BulkIndexByScrollResponseMatcher matcher() { return new BulkIndexByScrollResponseMatcher(); } + + protected int expectedSlices(Slices requestSlices, Collection indices) { + int slicesConfigured; + if (requestSlices.isAuto()) { + int leastNumShards = Collections.min(indices.stream() + .map(sourceIndex -> getNumShards(sourceIndex).numPrimaries) + .collect(Collectors.toList())); + slicesConfigured = Math.min(leastNumShards, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); + } else { + slicesConfigured = requestSlices.number(); + } + + if (slicesConfigured > 1) { + return slicesConfigured; + } else { + return 0; + } + } + + protected int expectedSlices(Slices slicesConfigured, String index) { + return expectedSlices(slicesConfigured, singleton(index)); + } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index e0af18a202a1b..59021edd295cc 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -19,8 +19,16 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.search.sort.SortOrder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.hasSize; @@ -64,36 +72,8 @@ public void testBasics() throws Exception { } public void testSlices() throws Exception { - indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"), - client().prepareIndex("test", "test", "2").setSource("foo", "a"), - client().prepareIndex("test", "test", "3").setSource("foo", "b"), - client().prepareIndex("test", "test", "4").setSource("foo", "c")); - assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 4); - assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion()); - assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); - - // Reindex all the docs - assertThat(updateByQuery().source("test").refresh(true).setSlices(Slices.of(5)).get(), matcher().updated(4).slices(hasSize(5))); - assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); - assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); - - // Now none of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).setSlices(Slices.of(5)).refresh(true).get(), - matcher().updated(0).slices(hasSize(5))); - assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); - assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); - - // Now half of them - assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).setSlices(Slices.of(5)).get(), - matcher().updated(2).slices(hasSize(5))); - assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); - assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); - assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion()); - assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); - } - - public void testSlicesAuto() throws Exception { - indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"), + indexRandom(true, + client().prepareIndex("test", "test", "1").setSource("foo", "a"), client().prepareIndex("test", "test", "2").setSource("foo", "a"), client().prepareIndex("test", "test", "3").setSource("foo", "b"), client().prepareIndex("test", "test", "4").setSource("foo", "c")); @@ -101,20 +81,20 @@ public void testSlicesAuto() throws Exception { assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); - NumShards numShards = getNumShards("test"); - int numSlicesExpected = numShards.numPrimaries > 1 - ? Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING) - : 0; + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(2, 10)); + int expectedSlices = expectedSlices(slices, "test"); // Reindex all the docs assertThat( updateByQuery() .source("test") .refresh(true) - .setSlices(Slices.AUTO).get(), + .setSlices(slices).get(), matcher() .updated(4) - .slices(hasSize(numSlicesExpected))); + .slices(hasSize(expectedSlices))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); @@ -123,11 +103,11 @@ public void testSlicesAuto() throws Exception { updateByQuery() .source("test") .filter(termQuery("foo", "no_match")) - .setSlices(Slices.AUTO) + .setSlices(slices) .refresh(true).get(), matcher() .updated(0) - .slices(hasSize(numSlicesExpected))); + .slices(hasSize(expectedSlices))); assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); @@ -137,13 +117,49 @@ public void testSlicesAuto() throws Exception { .source("test") .filter(termQuery("foo", "a")) .refresh(true) - .setSlices(Slices.AUTO).get(), + .setSlices(slices).get(), matcher() .updated(2) - .slices(hasSize(numSlicesExpected))); + .slices(hasSize(expectedSlices))); assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion()); assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion()); } + + public void testMultipleSources() throws Exception { + int sourceIndices = between(2, 5); + + Map> docs = new HashMap<>(); + for (int sourceIndex = 0; sourceIndex < sourceIndices; sourceIndex++) { + String indexName = "test" + sourceIndex; + docs.put(indexName, new ArrayList<>()); + int numDocs = between(5, 15); + for (int i = 0; i < numDocs; i++) { + docs.get(indexName).add(client().prepareIndex(indexName, "test", Integer.toString(i)).setSource("foo", "a")); + } + } + + List allDocs = docs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + indexRandom(true, allDocs); + for (Map.Entry> entry : docs.entrySet()) { + assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); + } + + Slices slices = randomBoolean() + ? Slices.AUTO + : Slices.of(between(1, 10)); + int expectedSlices = expectedSlices(slices, docs.keySet()); + + String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); + BulkByScrollResponse response = updateByQuery().source(sourceIndexNames).refresh(true).setSlices(slices).get(); + assertThat(response, matcher().updated(allDocs.size()).slices(hasSize(expectedSlices))); + + for (Map.Entry> entry : docs.entrySet()) { + String index = entry.getKey(); + List indexDocs = entry.getValue(); + int randomDoc = between(0, indexDocs.size() - 1); + assertEquals(2, client().prepareGet(index, "test", Integer.toString(randomDoc)).get().getVersion()); + } + } } From fd54d6499e9d2f0a0fbf72aa7f76ebfa440600bd Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 1 Aug 2017 18:32:32 -0700 Subject: [PATCH 13/21] wip #24547 move common slicing code into helper --- .../BulkByScrollParallelizationHelper.java | 55 ++++++++++++++----- .../reindex/TransportDeleteByQueryAction.java | 25 +++------ .../index/reindex/TransportReindexAction.java | 37 +++++-------- .../reindex/TransportUpdateByQueryAction.java | 25 +++------ 4 files changed, 75 insertions(+), 67 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 2b592701d3f1b..ef80cd315047b 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -24,12 +24,12 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.UidFieldMapper; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.tasks.TaskManager; import java.util.Arrays; import java.util.Collections; @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -44,23 +45,49 @@ */ class BulkByScrollParallelizationHelper { - public static final int AUTO_SLICE_CEILING = 20; + static final int AUTO_SLICE_CEILING = 20; private BulkByScrollParallelizationHelper() {} + static > void yourNameHere( //todo rename + Request request, + BulkByScrollTask task, + Action action, + ActionListener listener, + Client client, + DiscoveryNode node, + Supplier> taskSupplier) { + + computeSlicing(client, request, listener, slices -> { + if (slices > 1) { + task.setParent(slices); + sendSubRequests(client, action, node.getId(), task, request, listener); + } else { + Integer sliceId = request.getSearchRequest().source().slice() == null + ? null + : request.getSearchRequest().source().slice().getId(); + task.setChild(sliceId, request.getRequestsPerSecond()); + taskSupplier.get().start(); + } + }); + } + /** * Figures out how many slices to use when handling this request. If the {@code request} has slices set as a number, that number * will be used. If set to {@code "auto"}, it will compute it from the source indices' properties. The given consumer is passed the * resulting number of slices in either case. */ - public static > void - computeSlicing(Client client, Request request, ActionListener listener, Consumer slicedBehavior) { + private static > void computeSlicing( + Client client, + Request request, + ActionListener listener, + Consumer slicedBehavior) { Slices slices = request.getSlices(); if (slices.isAuto()) { client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( - response -> slicedBehavior.accept(sliceBasedOnShards(response)), - exception -> listener.onFailure(exception) + response -> slicedBehavior.accept(countSlicesBasedOnShards(response)), + listener::onFailure )); } else { slicedBehavior.accept(request.getSlices().number()); @@ -68,7 +95,7 @@ private BulkByScrollParallelizationHelper() {} } - private static int sliceBasedOnShards(ClusterSearchShardsResponse response) { + private static int countSlicesBasedOnShards(ClusterSearchShardsResponse response) { Map countsByIndex = Arrays.stream(response.getGroups()).collect(Collectors.toMap( group -> group.getShardId().getIndex(), __ -> 1, @@ -79,12 +106,14 @@ private static int sliceBasedOnShards(ClusterSearchShardsResponse response) { return Math.min(leastShards, AUTO_SLICE_CEILING); } - public static > void startSlices(Client client, TaskManager taskManager, - Action action, - String localNodeId, - BulkByScrollTask task, - Request request, - ActionListener listener) { + private static > void sendSubRequests( + Client client, + Action action, + String localNodeId, + BulkByScrollTask task, + Request request, + ActionListener listener) { + ParentBulkByScrollWorker worker = task.getParentWorker(); int totalSlices = worker.getSlices(); TaskId parentTaskId = new TaskId(localNodeId, task.getId()); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index d0704aa65826b..2ccd16135e2e6 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -51,24 +51,17 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { - BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { - - BulkByScrollTask bulkTask = (BulkByScrollTask) task; - - if (slices > 1) { - bulkTask.setParent(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, - clusterService.localNode().getId(), bulkTask, request, listener); - } else { - Integer sliceId = request.getSearchRequest().source().slice() == null - ? null - : request.getSearchRequest().source().slice().getId(); - bulkTask.setChild(sliceId, request.getRequestsPerSecond()); + BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + BulkByScrollParallelizationHelper.yourNameHere(request, bulkByScrollTask, DeleteByQueryAction.INSTANCE, listener, client, + clusterService.localNode(), + () -> { ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), bulkTask); - new AsyncDeleteByQueryAction(bulkTask, logger, client, threadPool, request, scriptService, state, listener).start(); + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), + bulkByScrollTask); + return new AsyncDeleteByQueryAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + listener); } - }); + ); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 16b3d7e19982b..1dadfda9162ca 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -109,29 +109,22 @@ public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFi @Override protected void doExecute(Task task, ReindexRequest request, ActionListener listener) { - BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { - - BulkByScrollTask bulkTask = (BulkByScrollTask) task; - - if (slices > 1) { - bulkTask.setParent(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, - clusterService.localNode().getId(), bulkTask, request, listener); - } else { - checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); - ClusterState state = clusterService.state(); - validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), - indexNameExpressionResolver, autoCreateIndex, state); - - Integer sliceId = request.getSearchRequest().source().slice() == null - ? null - : request.getSearchRequest().source().slice().getId(); - bulkTask.setChild(sliceId, request.getRequestsPerSecond()); - ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkTask); - new AsyncIndexBySearchAction(bulkTask, logger, assigningClient, threadPool, request, scriptService, state, - listener).start(); + checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo()); + ClusterState state = clusterService.state(); + validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), + indexNameExpressionResolver, autoCreateIndex, state); + + BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + + BulkByScrollParallelizationHelper.yourNameHere(request, bulkByScrollTask, ReindexAction.INSTANCE, listener, client, + clusterService.localNode(), + () -> { + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), + bulkByScrollTask); + return new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + listener); } - }); + ); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index a659b10dc4b43..8d0039182d03d 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -64,24 +64,17 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { - BulkByScrollParallelizationHelper.computeSlicing(client, request, listener, slices -> { - - BulkByScrollTask bulkTask = (BulkByScrollTask) task; - - if (slices > 1) { - bulkTask.setParent(slices); - BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, - clusterService.localNode().getId(), bulkTask, request, listener); - } else { - Integer sliceId = request.getSearchRequest().source().slice() == null - ? null - : request.getSearchRequest().source().slice().getId(); - bulkTask.setChild(sliceId, request.getRequestsPerSecond()); + BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + BulkByScrollParallelizationHelper.yourNameHere(request, bulkByScrollTask, UpdateByQueryAction.INSTANCE, listener, client, + clusterService.localNode(), + () -> { ClusterState state = clusterService.state(); - ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); - new AsyncIndexBySearchAction(bulkTask, logger, client, threadPool, request, scriptService, state, listener).start(); + ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), + bulkByScrollTask); + return new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + listener); } - }); + ); } @Override From 132d3c0a79eb4d8cacdb3cd8e3341ae18af7c15d Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Wed, 2 Aug 2017 10:41:49 -0700 Subject: [PATCH 14/21] wip #24547 update docs --- docs/reference/docs/delete-by-query.asciidoc | 55 +++++++++++++------- docs/reference/docs/reindex.asciidoc | 52 +++++++++++------- docs/reference/docs/update-by-query.asciidoc | 52 +++++++++++------- 3 files changed, 101 insertions(+), 58 deletions(-) diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index 1e26aac6d612f..eeb467653e348 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -338,12 +338,20 @@ query takes effect immediately but rethrotting that slows down the query will take effect on after completing the current batch. This prevents scroll timeouts. +[float] +[[docs-delete-by-query-slice]] +=== Slicing + +Delete-by-query supports <> to parallelize the deleting process. +This parallelization can improve efficiency and provide a convenient way to +break the request down into smaller parts. + [float] [[docs-delete-by-query-manual-slice]] -=== Manually slicing +==== Manually slicing -Delete-by-query supports <> allowing you to manually parallelize -the process relatively easily: +Slice a delete-by-query manually by providing a slice id and total number of +slices to each request: [source,js] ---------------------------------------------------------------- @@ -412,10 +420,11 @@ Which results in a sensible `total` like this one: [float] [[docs-delete-by-query-automatic-slice]] -=== Automatic slicing +==== Automatic slicing You can also let delete-by-query automatically parallelize using -<> to slice on `_uid`: +<> to slice on `_uid`. Use `slices` to specify the number of +slices to use: [source,js] ---------------------------------------------------------------- @@ -463,6 +472,11 @@ Which results in a sensible `total` like this one: ---------------------------------------------------------------- // TESTRESPONSE +Setting `slices` to `auto` will let Elasticsearch choose the number of slices +to use. This setting will use one slice per shard, up to a certain limit. If +there are multiple source indices, it will choose the number of slices based +on the index with the smallest number of shards. + Adding `slices` to `_delete_by_query` just automates the manual process used in the section above, creating sub-requests which means it has some quirks: @@ -489,18 +503,19 @@ though these are all taken at approximately the same time. [float] [[docs-delete-by-query-picking-slices]] -=== Picking the number of slices - -At this point we have a few recommendations around the number of `slices` to -use (the `max` parameter in the slice API if manually parallelizing): - -* Don't use large numbers. `500` creates fairly massive CPU thrash. -* It is more efficient from a query performance standpoint to use some multiple -of the number of shards in the source index. -* Using exactly as many shards as are in the source index is the most efficient -from a query performance standpoint. -* Indexing performance should scale linearly across available resources with -the number of `slices`. -* Whether indexing or query performance dominates that process depends on lots -of factors like the documents being reindexed and the cluster doing the -reindexing. +===== Picking the number of slices + +If slicing automatically, setting `slices` to `auto` will choose a good number +for most indices. + +Query performance is most efficient when the number of `slices` is equal to the +number of shards in the index. If that number is large, (for example, +500) choose a lower number as too many `slices` will hurt performance. Setting +`slices` higher than the number of shards generally does not improve efficiency +and adds overhead. + +Indexing performance scales linearly across available resources with the +number of slices. + +Whether query or indexing performance dominates the runtime depends on the +documents being reindexed and cluster resources. diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index eaefa1a07a2e5..9ca48155dcff6 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -783,11 +783,19 @@ and it'll look like: Or you can search by `tag` or whatever you want. +[float] +[[docs-reindex-slice]] +=== Slicing + +Reindex supports <> to parallelize the reindexing process. +This parallelization can improve efficiency and provide a convenient way to +break the request down into smaller parts. + [float] [[docs-reindex-manual-slice]] ==== Manual slicing -Reindex supports <>, allowing you to manually parallelize the -process relatively easily: +Slice a reindex request manually by providing a slice id and total number of +slices to each request: [source,js] ---------------------------------------------------------------- @@ -845,10 +853,10 @@ Which results in a sensible `total` like this one: [float] [[docs-reindex-automatic-slice]] -=== Automatic slicing +==== Automatic slicing You can also let reindex automatically parallelize using <> to -slice on `_uid`: +slice on `_uid`. Use `slices` to specify the number of slices to use: [source,js] ---------------------------------------------------------------- @@ -886,6 +894,11 @@ Which results in a sensible `total` like this one: ---------------------------------------------------------------- // TESTRESPONSE +Setting `slices` to `auto` will let Elasticsearch choose the number of slices +to use. This setting will use one slice per shard, up to a certain limit. If +there are multiple source indices, it will choose the number of slices based +on the index with the smallest number of shards. + Adding `slices` to `_reindex` just automates the manual process used in the section above, creating sub-requests which means it has some quirks: @@ -911,21 +924,22 @@ though these are all taken at approximately the same time. [float] [[docs-reindex-picking-slices]] -=== Picking the number of slices - -At this point we have a few recommendations around the number of `slices` to -use (the `max` parameter in the slice API if manually parallelizing): - -* Don't use large numbers. `500` creates fairly massive CPU thrash. -* It is more efficient from a query performance standpoint to use some multiple -of the number of shards in the source index. -* Using exactly as many shards as are in the source index is the most efficient -from a query performance standpoint. -* Indexing performance should scale linearly across available resources with -the number of `slices`. -* Whether indexing or query performance dominates that process depends on lots -of factors like the documents being reindexed and the cluster doing the -reindexing. +===== Picking the number of slices + +If slicing automatically, setting `slices` to `auto` will choose a good number +for most indices. + +Query performance is most efficient when the number of `slices` is equal to the +number of shards in the index. If that number is large, (for example, +500) choose a lower number as too many `slices` will hurt performance. Setting +`slices` higher than the number of shards generally does not improve efficiency +and adds overhead. + +Indexing performance scales linearly across available resources with the +number of slices. + +Whether query or indexing performance dominates the runtime depends on the +documents being reindexed and cluster resources. [float] === Reindex daily indices diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index 28c250dcfe193..65fa5d5b73f47 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -403,11 +403,19 @@ query takes effect immediately but rethrotting that slows down the query will take effect on after completing the current batch. This prevents scroll timeouts. +[float] +[[docs-update-by-query-slice]] +=== Slicing + +Update-by-query supports <> to parallelize the updating process. +This parallelization can improve efficiency and provide a convenient way to +break the request down into smaller parts. + [float] [[docs-update-by-query-manual-slice]] ==== Manual slicing -Update-by-query supports <> allowing you to manually parallelize -the process relatively easily: +Slice an update-by-query manually by providing a slice id and total number of +slices to each request: [source,js] ---------------------------------------------------------------- @@ -459,10 +467,11 @@ Which results in a sensible `total` like this one: [float] [[docs-update-by-query-automatic-slice]] -=== Automatic slicing +==== Automatic slicing You can also let update-by-query automatically parallelize using -<> to slice on `_uid`: +<> to slice on `_uid`. Use `slices` to specify the number of +slices to use: [source,js] ---------------------------------------------------------------- @@ -497,6 +506,11 @@ Which results in a sensible `total` like this one: ---------------------------------------------------------------- // TESTRESPONSE +Setting `slices` to `auto` will let Elasticsearch choose the number of slices +to use. This setting will use one slice per shard, up to a certain limit. If +there are multiple source indices, it will choose the number of slices based +on the index with the smallest number of shards. + Adding `slices` to `_update_by_query` just automates the manual process used in the section above, creating sub-requests which means it has some quirks: @@ -523,22 +537,22 @@ though these are all taken at approximately the same time. [float] [[docs-update-by-query-picking-slices]] -=== Picking the number of slices - -At this point we have a few recommendations around the number of `slices` to -use (the `max` parameter in the slice API if manually parallelizing): - -* Don't use large numbers. `500` creates fairly massive CPU thrash. -* It is more efficient from a query performance standpoint to use some multiple -of the number of shards in the source index. -* Using exactly as many shards as are in the source index is the most efficient -from a query performance standpoint. -* Indexing performance should scale linearly across available resources with -the number of `slices`. -* Whether indexing or query performance dominates that process depends on lots -of factors like the documents being reindexed and the cluster doing the -reindexing. +===== Picking the number of slices + +If slicing automatically, setting `slices` to `auto` will choose a good number +for most indices. + +Query performance is most efficient when the number of `slices` is equal to the +number of shards in the index. If that number is large, (for example, +500) choose a lower number as too many `slices` will hurt performance. Setting +`slices` higher than the number of shards generally does not improve efficiency +and adds overhead. + +Indexing performance scales linearly across available resources with the +number of slices. +Whether query or indexing performance dominates the runtime depends on the +documents being reindexed and cluster resources. [float] [[picking-up-a-new-property]] From 22029881c9cd5fd88650cd96fb36e1cdc5eeed99 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Wed, 2 Aug 2017 10:42:12 -0700 Subject: [PATCH 15/21] wip #24547 simplify parallelization helper --- .../BulkByScrollParallelizationHelper.java | 66 +++++++++++-------- .../reindex/TransportDeleteByQueryAction.java | 2 +- .../index/reindex/TransportReindexAction.java | 2 +- .../reindex/TransportUpdateByQueryAction.java | 2 +- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index ef80cd315047b..94c2f4bd6a675 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -36,7 +36,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -49,7 +48,17 @@ class BulkByScrollParallelizationHelper { private BulkByScrollParallelizationHelper() {} - static > void yourNameHere( //todo rename + /** + * Takes an action created by a {@link BulkByScrollTask} and runs it with regard to whether the request is sliced or not. + * + * If the request is not sliced (e.g. the number of slices is 1), the action from the given {@link Supplier} will be started on the + * local node. If the request is sliced (e.g. the number of slices is more than 1), then a subrequest will be created for each slice + * and sent. + * + * If slices are set as {@code "auto"}, this class will resolve that to a specific number based on characteristics of the source + * indices. A request with {@code "auto"} slices may end up being sliced or unsliced. + */ + static > void startSlicedAction( Request request, BulkByScrollTask task, Action action, @@ -58,41 +67,40 @@ static > void yourNameHere( DiscoveryNode node, Supplier> taskSupplier) { - computeSlicing(client, request, listener, slices -> { - if (slices > 1) { - task.setParent(slices); - sendSubRequests(client, action, node.getId(), task, request, listener); - } else { - Integer sliceId = request.getSearchRequest().source().slice() == null - ? null - : request.getSearchRequest().source().slice().getId(); - task.setChild(sliceId, request.getRequestsPerSecond()); - taskSupplier.get().start(); - } - }); - } - - /** - * Figures out how many slices to use when handling this request. If the {@code request} has slices set as a number, that number - * will be used. If set to {@code "auto"}, it will compute it from the source indices' properties. The given consumer is passed the - * resulting number of slices in either case. - */ - private static > void computeSlicing( - Client client, - Request request, - ActionListener listener, - Consumer slicedBehavior) { - Slices slices = request.getSlices(); if (slices.isAuto()) { client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( - response -> slicedBehavior.accept(countSlicesBasedOnShards(response)), + response -> { + int actualNumSlices = countSlicesBasedOnShards(response); + sliceConditionally(request, task, action, listener, client, node, taskSupplier, actualNumSlices); + }, listener::onFailure )); } else { - slicedBehavior.accept(request.getSlices().number()); + sliceConditionally(request, task, action, listener, client, node, taskSupplier, slices.number()); } + } + private static > void sliceConditionally( + Request request, + BulkByScrollTask task, + Action action, + ActionListener listener, + Client client, + DiscoveryNode node, + Supplier> taskSupplier, + int slices) { + + if (slices > 1) { + task.setParent(slices); + sendSubRequests(client, action, node.getId(), task, request, listener); + } else { + Integer sliceId = request.getSearchRequest().source().slice() == null + ? null + : request.getSearchRequest().source().slice().getId(); + task.setChild(sliceId, request.getRequestsPerSecond()); + taskSupplier.get().start(); + } } private static int countSlicesBasedOnShards(ClusterSearchShardsResponse response) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 2ccd16135e2e6..719dc34efd9e7 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -52,7 +52,7 @@ public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; - BulkByScrollParallelizationHelper.yourNameHere(request, bulkByScrollTask, DeleteByQueryAction.INSTANCE, listener, client, + BulkByScrollParallelizationHelper.startSlicedAction(request, bulkByScrollTask, DeleteByQueryAction.INSTANCE, listener, client, clusterService.localNode(), () -> { ClusterState state = clusterService.state(); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 1dadfda9162ca..e2c3c7163fddd 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -116,7 +116,7 @@ protected void doExecute(Task task, ReindexRequest request, ActionListener { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 8d0039182d03d..fd4568c27d47d 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -65,7 +65,7 @@ public TransportUpdateByQueryAction(Settings settings, ThreadPool threadPool, Ac @Override protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; - BulkByScrollParallelizationHelper.yourNameHere(request, bulkByScrollTask, UpdateByQueryAction.INSTANCE, listener, client, + BulkByScrollParallelizationHelper.startSlicedAction(request, bulkByScrollTask, UpdateByQueryAction.INSTANCE, listener, client, clusterService.localNode(), () -> { ClusterState state = clusterService.state(); From 3336d983ad5433fe93ef21007d1466b376092c6b Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Wed, 2 Aug 2017 12:50:08 -0700 Subject: [PATCH 16/21] wip #24547 fix some unit tests At this point everything passes --- .../index/reindex/BulkByScrollTask.java | 6 ++++- .../reindex/ChildBulkByScrollWorker.java | 3 +++ .../reindex/DeleteByQueryBasicTests.java | 4 ++-- .../index/reindex/ReindexBasicTests.java | 4 ++-- .../index/reindex/ReindexTestCase.java | 23 +++++++++++++++---- .../index/reindex/RethrottleTests.java | 8 +------ .../index/reindex/RoundTripTests.java | 2 +- .../reindex/UpdateByQueryBasicTests.java | 4 ++-- 8 files changed, 34 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 2a1f3c35e3fd0..42986af5d80c4 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -48,10 +48,14 @@ * * When the request is sliced, this task can either represent a parent coordinating task (using {@link BulkByScrollTask#setParent(int)}) or * a child task that performs search queries (using {@link BulkByScrollTask#setChild(Integer, float)}). + * + * We don't always know if this task will be a parent or child task when it's created, because if slices is set to "auto" it may + * be either depending on the number of shards in the source indices. We figure that out when the request is handled and set it on this + * class with {@link #setParent(int)} or {@link #setChild(Integer, float)}. */ public class BulkByScrollTask extends CancellableTask { - private ParentBulkByScrollWorker parentWorker; // todo give this a better name (maybe task strategy or behavior) + private ParentBulkByScrollWorker parentWorker; private ChildBulkByScrollWorker childWorker; public BulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java index 52f30bd6a12b9..4ff4cf1bff00e 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java @@ -37,6 +37,9 @@ import static java.lang.Math.round; import static org.elasticsearch.common.unit.TimeValue.timeValueNanos; +/** + * Task behavior for {@link BulkByScrollTask} that does the actual work of querying and indexing + */ public class ChildBulkByScrollWorker implements SuccessfullyProcessed { private static final Logger logger = Loggers.getLogger(ChildBulkByScrollWorker.class); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index eae19a4088b3c..95109cd851651 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -241,7 +241,7 @@ public void testSlices() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(2, 10)); - int expectedSlices = expectedSlices(slices, "test"); + int expectedSlices = expectedSliceStatuses(slices, "test"); // Deletes the two docs that matches "foo:a" assertThat( @@ -290,7 +290,7 @@ public void testMultipleSources() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(1, 10)); - int expectedSlices = expectedSlices(slices, docs.keySet()); + int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index c8bccb8bea17c..ccd5f08d39747 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -104,7 +104,7 @@ public void testCopyManyWithSlices() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(2, 10)); - int expectedSlices = expectedSlices(slices, "source"); + int expectedSlices = expectedSliceStatuses(slices, "source"); // Copy all the docs ReindexRequestBuilder copy = reindex().source("source").destination("dest", "type").refresh(true).setSlices(slices); @@ -147,7 +147,7 @@ public void testMultipleSources() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(1, 10)); - int expectedSlices = expectedSlices(slices, docs.keySet()); + int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); ReindexRequestBuilder request = reindex() diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java index 873c24e68d7af..7e32a9f3fefe6 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java @@ -66,16 +66,29 @@ public static BulkIndexByScrollResponseMatcher matcher() { return new BulkIndexByScrollResponseMatcher(); } + /** + * Figures out how many slices the request handling will use + */ protected int expectedSlices(Slices requestSlices, Collection indices) { - int slicesConfigured; if (requestSlices.isAuto()) { int leastNumShards = Collections.min(indices.stream() .map(sourceIndex -> getNumShards(sourceIndex).numPrimaries) .collect(Collectors.toList())); - slicesConfigured = Math.min(leastNumShards, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); + return Math.min(leastNumShards, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); } else { - slicesConfigured = requestSlices.number(); + return requestSlices.number(); } + } + + protected int expectedSlices(Slices requestSlices, String index) { + return expectedSlices(requestSlices, singleton(index)); + } + + /** + * Figures out how many slice statuses to expect in the response + */ + protected int expectedSliceStatuses(Slices requestSlices, Collection indices) { + int slicesConfigured = expectedSlices(requestSlices, indices); if (slicesConfigured > 1) { return slicesConfigured; @@ -84,7 +97,7 @@ protected int expectedSlices(Slices requestSlices, Collection indices) { } } - protected int expectedSlices(Slices slicesConfigured, String index) { - return expectedSlices(slicesConfigured, singleton(index)); + protected int expectedSliceStatuses(Slices slicesConfigured, String index) { + return expectedSliceStatuses(slicesConfigured, singleton(index)); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index 33c57804be7ff..8eb613259dbd2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -86,13 +86,7 @@ private void testCase(AbstractBulkByScrollRequestBuilder request, String a */ createIndex("test"); - int numSlices; - if (request.request().getSlices().isAuto()) { - NumShards numShards = getNumShards("test"); - numSlices = Math.min(numShards.numPrimaries, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); - } else { - numSlices = request.request().getSlices().number(); - } + int numSlices = expectedSlices(request.request().getSlices(), "test"); List docs = new ArrayList<>(); for (int i = 0; i < numSlices * 10; i++) { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index d97f57aab489b..b2a31ef789209 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -142,7 +142,7 @@ private void randomRequest(AbstractBulkByScrollRequest request) { Slices slices = randomBoolean() ? Slices.AUTO - : Slices.of(between(0, Integer.MAX_VALUE)); + : Slices.of(between(1, Integer.MAX_VALUE)); request.setSlices(slices); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index 59021edd295cc..17381ff64d293 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -84,7 +84,7 @@ public void testSlices() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(2, 10)); - int expectedSlices = expectedSlices(slices, "test"); + int expectedSlices = expectedSliceStatuses(slices, "test"); // Reindex all the docs assertThat( @@ -149,7 +149,7 @@ public void testMultipleSources() throws Exception { Slices slices = randomBoolean() ? Slices.AUTO : Slices.of(between(1, 10)); - int expectedSlices = expectedSlices(slices, docs.keySet()); + int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); BulkByScrollResponse response = updateByQuery().source(sourceIndexNames).refresh(true).setSlices(slices).get(); From 25518ef86416fb652a0cf0c9336d9d1dfcbc75e5 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Mon, 7 Aug 2017 15:46:44 -0700 Subject: [PATCH 17/21] wip #24547 fixes for code review * Move back to integer type for slices * Better method names in BulkByScrollTask * BWC only to 6.1.0 and later, tests included --- .../reindex/AbstractBulkByScrollRequest.java | 34 ++-- .../AbstractBulkByScrollRequestBuilder.java | 2 +- .../index/reindex/BulkByScrollResponse.java | 3 +- .../index/reindex/BulkByScrollTask.java | 55 ++++--- .../index/reindex/ReindexRequest.java | 4 +- .../elasticsearch/index/reindex/Slices.java | 152 ------------------ .../AbstractBulkByScrollRequestTestCase.java | 6 +- .../reindex/ChildBulkByScrollWorkerTests.java | 4 +- .../ParentBulkByScrollWorkerTests.java | 4 +- .../index/reindex/ReindexRequestTests.java | 12 +- .../AbstractAsyncBulkByScrollAction.java | 4 +- .../AbstractBaseReindexRestHandler.java | 29 +++- .../BulkByScrollParallelizationHelper.java | 11 +- .../reindex/TransportRethrottleAction.java | 8 +- .../reindex/AsyncBulkByScrollActionTests.java | 4 +- .../index/reindex/CancelTests.java | 22 +-- .../reindex/DeleteByQueryBasicTests.java | 8 +- .../index/reindex/ReindexBasicTests.java | 8 +- .../index/reindex/ReindexTestCase.java | 24 ++- .../index/reindex/RethrottleTests.java | 8 - .../index/reindex/RoundTripTests.java | 51 +++--- .../TransportRethrottleActionTests.java | 6 +- .../reindex/UpdateByQueryBasicTests.java | 8 +- .../test/delete_by_query/20_validation.yml | 2 +- .../test/reindex/20_validation.yml | 2 +- .../test/update_by_query/20_validation.yml | 2 +- ...stractAsyncBulkByScrollActionTestCase.java | 2 +- 27 files changed, 172 insertions(+), 303 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/index/reindex/Slices.java diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 312d9de7d64ad..afd8181b88fc5 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -45,6 +45,10 @@ public abstract class AbstractBulkByScrollRequest 1) { - throw new IllegalArgumentException("Attempting to send sliced reindex-style request to a node that doesn't support " - + "it. Version is [" + out.getVersion() + "] but must be [" + Version.V_5_1_1 + "]"); + if (slices == AUTO_SLICES) { + throw new IllegalArgumentException("Slices set as \"auto\" are not supported before version [" + Version.V_6_1_0 + "]. " + + "Found version [" + out.getVersion() + "]"); + } else { + out.writeVInt(slices); } } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java index 49317714ef310..e3c5bd2197a94 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java @@ -145,7 +145,7 @@ public Self setShouldStoreResult(boolean shouldStoreResult) { /** * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. */ - public Self setSlices(Slices slices) { + public Self setSlices(int slices) { request.setSlices(slices); return self(); } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java index 0b70c4e7a7b29..a4731e1781aeb 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollResponse.java @@ -190,8 +190,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder.append(getClass().getSimpleName()); - builder.append("["); + builder.append(getClass().getSimpleName()).append("["); builder.append("took=").append(took).append(','); builder.append("timed_out=").append(timedOut).append(','); status.innerToString(builder); diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index 42986af5d80c4..d0526fe289877 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -46,12 +46,13 @@ * * When the request is not sliced, this task is the only task created, and starts an action to perform search requests. * - * When the request is sliced, this task can either represent a parent coordinating task (using {@link BulkByScrollTask#setParent(int)}) or - * a child task that performs search queries (using {@link BulkByScrollTask#setChild(Integer, float)}). + * When the request is sliced, this task can either represent a parent coordinating task (using + * {@link BulkByScrollTask#setSliceChildren(int)}) or a child task that performs search queries (using + * {@link BulkByScrollTask#setSliceChild(float, Integer)}). * * We don't always know if this task will be a parent or child task when it's created, because if slices is set to "auto" it may * be either depending on the number of shards in the source indices. We figure that out when the request is handled and set it on this - * class with {@link #setParent(int)} or {@link #setChild(Integer, float)}. + * class with {@link #setSliceChildren(int)} or {@link #setSliceChild(float, Integer)}. */ public class BulkByScrollTask extends CancellableTask { @@ -64,11 +65,11 @@ public BulkByScrollTask(long id, String type, String action, String description, @Override public BulkByScrollTask.Status getStatus() { - if (isParent()) { + if (isSlicesParent()) { return parentWorker.getStatus(); } - if (isChild()) { + if (isSliceChild()) { return childWorker.getStatus(); } @@ -82,18 +83,18 @@ private BulkByScrollTask.Status emptyStatus() { /** * Returns true if this task is a parent of other sliced tasks. False otherwise */ - public boolean isParent() { + public boolean isSlicesParent() { return parentWorker != null; } /** * Sets this task to be a parent task for {@code slices} sliced subtasks */ - public void setParent(int slices) { - if (isParent()) { + public void setSliceChildren(int slices) { + if (isSlicesParent()) { throw new IllegalStateException("Parent worker is already set"); } - if (isChild()) { + if (isSliceChild()) { throw new IllegalStateException("Worker is already set as child"); } @@ -104,8 +105,8 @@ public void setParent(int slices) { * Returns the worker object that tracks the state of sliced subtasks. Throws IllegalStateException if this task is not set to be * a parent task. */ - public ParentBulkByScrollWorker getParentWorker() { - if (!isParent()) { + public ParentBulkByScrollWorker getSlicesParentWorker() { + if (!isSlicesParent()) { throw new IllegalStateException("This task is not set as a parent worker"); } return parentWorker; @@ -114,20 +115,28 @@ public ParentBulkByScrollWorker getParentWorker() { /** * Returns true if this task is a child task that performs search requests. False otherwise */ - public boolean isChild() { + public boolean isSliceChild() { return childWorker != null; } /** - * Sets this task to be a child task that performs search requests - * @param sliceId If this is is a sliced task, which slice number this task corresponds to + * Sets this task to be a child task that performs search requests, when the request is not sliced. * @param requestsPerSecond How many search requests per second this task should make */ - public void setChild(Integer sliceId, float requestsPerSecond) { - if (isChild()) { + public void setSliceChild(float requestsPerSecond) { + setSliceChild(requestsPerSecond, null); + } + + /** + * Sets this task to be a slice child task that performs search requests + * @param requestsPerSecond How many search requests per second this task should make + * @param sliceId If this is is a sliced task, which slice number this task corresponds to + */ + public void setSliceChild(float requestsPerSecond, Integer sliceId) { + if (isSliceChild()) { throw new IllegalStateException("Child worker is already set"); } - if (isParent()) { + if (isSlicesParent()) { throw new IllegalStateException("Worker is already set as child"); } @@ -138,8 +147,8 @@ public void setChild(Integer sliceId, float requestsPerSecond) { * Returns the worker object that manages sending search requests. Throws IllegalStateException if this task is not set to be a * child task. */ - public ChildBulkByScrollWorker getChildWorker() { - if (!isChild()) { + public ChildBulkByScrollWorker getSliceChildWorker() { + if (!isSliceChild()) { throw new IllegalStateException("This task is not set as a child worker"); } return childWorker; @@ -147,12 +156,12 @@ public ChildBulkByScrollWorker getChildWorker() { @Override public void onCancelled() { - if (isParent()) { - // do nothing - } else if (isChild()) { + if (isSlicesParent()) { + // The task cancellation task automatically finds children and cancels them, nothing extra to do + } else if (isSliceChild()) { childWorker.handleCancel(); } else { - throw new IllegalStateException("This task's worker is not set"); + throw new IllegalStateException("This task has not had its sliced state initialized and doesn't know how to cancel itself"); } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java index 29ccb79d03085..276c455915323 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java @@ -94,8 +94,8 @@ public ActionRequestValidationException validate() { if (getSearchRequest().source().query() != null) { e = addValidationError("reindex from remote sources should use RemoteInfo's query instead of source's query", e); } - if (getSlices().isAuto() || getSlices().number() > 1) { - e = addValidationError("reindex from remote sources doesn't support workers > 1 but was [" + getSlices() + "]", e); + if (getSlices() == AbstractBulkByScrollRequest.AUTO_SLICES || getSlices() > 1) { + e = addValidationError("reindex from remote sources doesn't support slices > 1 but was [" + getSlices() + "]", e); } } return e; diff --git a/core/src/main/java/org/elasticsearch/index/reindex/Slices.java b/core/src/main/java/org/elasticsearch/index/reindex/Slices.java deleted file mode 100644 index 0c845d5c790e5..0000000000000 --- a/core/src/main/java/org/elasticsearch/index/reindex/Slices.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.reindex; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -/** - * Represents the setting for the number of slices used by a BulkByScrollRequest. Valid values are positive integers and "auto". The - * "auto" setting defers choosing the number of slices to the request handler. - */ -public final class Slices implements ToXContent, Writeable { - - private static final int AUTO_COUNT = -1; - - public static final String FIELD_NAME = "slices"; - public static final String AUTO_VALUE = "auto"; - - public static final Slices AUTO = new Slices(AUTO_COUNT); - public static final Slices ONE = of(1); - public static final Slices DEFAULT = ONE; - - private final int count; - - private Slices(int count) { - this.count = count; - } - - public Slices(StreamInput stream) throws IOException { - count = stream.readVInt(); - } - - /** - * Creates a new {@link Slices} as a concrete number. The given number must be a positive integer. - */ - public static Slices of(int count) { - if (count < 1) { - throw new IllegalArgumentException("[slices] must be at least 1"); - } - return new Slices(count); - } - - /** - * Parse a string to a valid {@link Slices} value - */ - public static Slices parse(String slicesString) { - Objects.requireNonNull(slicesString); - - if (AUTO_VALUE.equals(slicesString)) { - return AUTO; - } - - try { - int slicesNumber = Integer.parseInt(slicesString); - return of(slicesNumber); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\""); - } - } - - /** - * Returns true if this value is set as "auto", false otherwise - */ - public boolean isAuto() { - return count == AUTO_COUNT; - } - - /** - * Returns true if this value is set as a number, false otherwise - */ - public boolean isNumber() { - return !isAuto(); - } - - /** - * Returns the number value this {@link Slices} has, if it is set as a number. Otherwise throws IllegalStateException - */ - public int number() { - if (isAuto()) { - throw new IllegalStateException("Slice count is set as \"auto\", not a number"); - } - return count; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(count); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (isAuto()) { - builder.field(FIELD_NAME, AUTO_VALUE); - } else { - builder.field(FIELD_NAME, number()); - } - return builder; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (obj.getClass() != getClass()) { - return false; - } - - Slices other = (Slices) obj; - return other.count == count; - } - - @Override - public int hashCode() { - return Objects.hash(count); - } - - @Override - public String toString() { - if (isAuto()) { - return "auto"; - } else { - return Integer.toString(count); - } - } -} diff --git a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java index 49e84683906dc..1111eb44241d3 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java @@ -47,13 +47,13 @@ public void testForSlice() { int actualSlices = between(2, 1000); if (randomBoolean()) { - original.setSlices(Slices.of(actualSlices)); + original.setSlices(actualSlices); } else { /* * In this case we pretend that the true number of slices will be randomSlices, since we're just verifying the request * behavior and not the slicing itself */ - original.setSlices(Slices.AUTO); + original.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES); } TaskId slicingTask = new TaskId(randomAlphaOfLength(5), randomLong()); @@ -66,7 +66,7 @@ public void testForSlice() { assertEquals(original.getRetryBackoffInitialTime(), forSliced.getRetryBackoffInitialTime()); assertEquals(original.getMaxRetries(), forSliced.getMaxRetries()); assertEquals("only the parent task should store results", false, forSliced.getShouldStoreResult()); - assertEquals("slice requests always have a single worker", Slices.ONE, forSliced.getSlices()); + assertEquals("slice requests always have a single worker", 1, forSliced.getSlices()); assertEquals("requests_per_second is split between all workers", original.getRequestsPerSecond() / actualSlices, forSliced.getRequestsPerSecond(), Float.MIN_NORMAL); assertEquals("size is split evenly between all workers", original.getSize() == AbstractBulkByScrollRequest.SIZE_ALL_MATCHES diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java index db25d736390db..d156822f5da5c 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java @@ -53,8 +53,8 @@ public class ChildBulkByScrollWorkerTests extends ESTestCase { @Before public void createTask() { task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setChild(null, Float.POSITIVE_INFINITY); - worker = task.getChildWorker(); + task.setSliceChild(Float.POSITIVE_INFINITY); + worker = task.getSliceChildWorker(); } public void testBasicData() { diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java index d0894327b27c3..7c208906140aa 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java @@ -43,8 +43,8 @@ public class ParentBulkByScrollWorkerTests extends ESTestCase { public void createTask() { slices = between(2, 50); task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setParent(slices); - worker = task.getParentWorker(); + task.setSliceChildren(slices); + worker = task.getSlicesParentWorker(); } public void testBasicData() { diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index ad0715c6584f3..8a7c7323348b0 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -45,24 +45,24 @@ public void testReindexFromRemoteDoesNotSupportSearchQuery() { e.getMessage()); } - public void testReindexFromRemoteDoesNotSupportWorkers() { + public void testReindexFromRemoteDoesNotSupportSlices() { ReindexRequest reindex = newRequest(); reindex.setRemoteInfo( new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"), null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); - reindex.setSlices(Slices.of(between(2, Integer.MAX_VALUE))); + reindex.setSlices(between(2, Integer.MAX_VALUE)); ActionRequestValidationException e = reindex.validate(); assertEquals( - "Validation Failed: 1: reindex from remote sources doesn't support workers > 1 but was [" + reindex.getSlices() + "];", + "Validation Failed: 1: reindex from remote sources doesn't support slices > 1 but was [" + reindex.getSlices() + "];", e.getMessage()); } - public void testNoSliceWithWorkers() { + public void testNoSliceBuilderSetWithSlicedRequest() { ReindexRequest reindex = newRequest(); reindex.getSearchRequest().source().slice(new SliceBuilder(0, 4)); - reindex.setSlices(Slices.of(between(2, Integer.MAX_VALUE))); + reindex.setSlices(between(2, Integer.MAX_VALUE)); ActionRequestValidationException e = reindex.validate(); - assertEquals("Validation Failed: 1: can't specify both slice and workers;", e.getMessage()); + assertEquals("Validation Failed: 1: can't set a specific single slice for this request and multiple slices;", e.getMessage()); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index 24321ef68a609..d6786333b8c48 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -126,10 +126,10 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par ActionListener listener, Settings settings) { this.task = task; - if (!task.isChild()) { + if (!task.isSliceChild()) { throw new IllegalArgumentException("Given task [" + task.getId() + "] must have a child worker"); } - this.worker = task.getChildWorker(); + this.worker = task.getSliceChildWorker(); this.logger = logger; this.client = client; diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java index 22bad846e3037..edf1e610ac780 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -29,6 +29,7 @@ import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.tasks.LoggingTaskListener; import org.elasticsearch.tasks.Task; @@ -91,9 +92,9 @@ protected Request setCommonOptions(RestRequest restRequest, Request request) { request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh())); request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout())); - String slices = restRequest.param(Slices.FIELD_NAME); + Integer slices = parseSlices(restRequest); if (slices != null) { - request.setSlices(Slices.parse(slices)); + request.setSlices(slices); } String waitForActiveShards = restRequest.param("wait_for_active_shards"); @@ -119,6 +120,30 @@ private RestChannelConsumer sendTask(String localNodeId, Task task) throws IOExc }; } + private static Integer parseSlices(RestRequest request) { + String slicesString = request.param("slices"); + if (slicesString == null) { + return null; + } + + if (slicesString.equals(AbstractBulkByScrollRequest.AUTO_SLICES_VALUE)) { + return AbstractBulkByScrollRequest.AUTO_SLICES; + } + + int slices; + try { + slices = Integer.parseInt(slicesString); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\"", e); + } + + if (slices < 1) { + throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\""); + } + + return slices; + } + /** * @return requests_per_second from the request as a float if it was on the request, null otherwise */ diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 94c2f4bd6a675..80f03789bc4d8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -67,8 +67,7 @@ static > void startSlicedAc DiscoveryNode node, Supplier> taskSupplier) { - Slices slices = request.getSlices(); - if (slices.isAuto()) { + if (request.getSlices() == AbstractBulkByScrollRequest.AUTO_SLICES) { client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( response -> { int actualNumSlices = countSlicesBasedOnShards(response); @@ -77,7 +76,7 @@ static > void startSlicedAc listener::onFailure )); } else { - sliceConditionally(request, task, action, listener, client, node, taskSupplier, slices.number()); + sliceConditionally(request, task, action, listener, client, node, taskSupplier, request.getSlices()); } } @@ -92,13 +91,13 @@ private static > void slice int slices) { if (slices > 1) { - task.setParent(slices); + task.setSliceChildren(slices); sendSubRequests(client, action, node.getId(), task, request, listener); } else { Integer sliceId = request.getSearchRequest().source().slice() == null ? null : request.getSearchRequest().source().slice().getId(); - task.setChild(sliceId, request.getRequestsPerSecond()); + task.setSliceChild(request.getRequestsPerSecond(), sliceId); taskSupplier.get().start(); } } @@ -122,7 +121,7 @@ private static > void sendS Request request, ActionListener listener) { - ParentBulkByScrollWorker worker = task.getParentWorker(); + ParentBulkByScrollWorker worker = task.getSlicesParentWorker(); int totalSlices = worker.getSlices(); TaskId parentTaskId = new TaskId(localNodeId, task.getId()); for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, totalSlices)) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index e1ee629d79894..444b2c9d1040c 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -60,12 +60,12 @@ protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, A static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { - if (task.isChild()) { + if (task.isSliceChild()) { rethrottleChildTask(logger, localNodeId, task, newRequestsPerSecond, listener); return; } - if (task.isParent()) { + if (task.isSlicesParent()) { rethrottleParentTask(logger, localNodeId, client, task, newRequestsPerSecond, listener); return; } @@ -75,7 +75,7 @@ static void rethrottle(Logger logger, String localNodeId, Client client, BulkByS private static void rethrottleParentTask(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { - final ParentBulkByScrollWorker parentWorker = task.getParentWorker(); + final ParentBulkByScrollWorker parentWorker = task.getSlicesParentWorker(); final int runningSubtasks = parentWorker.runningSliceSubTasks(); if (runningSubtasks > 0) { @@ -99,7 +99,7 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie private static void rethrottleChildTask(Logger logger, String localNodeId, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond); - task.getChildWorker().rethrottle(newRequestsPerSecond); + task.getSliceChildWorker().rethrottle(newRequestsPerSecond); listener.onResponse(task.taskInfo(localNodeId, true)); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 7a6c8358c893b..62f2c02f01162 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -143,8 +143,8 @@ public void setupForTest() { scrollId = null; taskManager = new TaskManager(Settings.EMPTY); testTask = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest); - testTask.setChild(null, testRequest.getRequestsPerSecond()); - worker = testTask.getChildWorker(); + testTask.setSliceChild(testRequest.getRequestsPerSecond()); + worker = testTask.getSliceChildWorker(); localNode = new DiscoveryNode("thenode", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); taskId = new TaskId(localNode.getId(), testTask.getId()); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java index 726e6898e08f4..3ad48d803a437 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java @@ -88,7 +88,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder createIndex(INDEX); // Total number of documents created for this test (~10 per primary shard per slice) - int numDocs = getNumShards(INDEX).numPrimaries * 10 * builder.request().getSlices().number(); + int numDocs = getNumShards(INDEX).numPrimaries * 10 * builder.request().getSlices(); ALLOWED_OPERATIONS.release(numDocs); indexRandom(true, false, true, IntStream.range(0, numDocs) @@ -104,8 +104,8 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder /* Allow a random number of the documents less the number of workers to be modified by the reindex action. That way at least one * worker is blocked. */ - int numModifiedDocs = randomIntBetween(builder.request().getSlices().number() * 2, numDocs); - ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices().number()); + int numModifiedDocs = randomIntBetween(builder.request().getSlices() * 2, numDocs); + ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices()); // Now execute the reindex action... ActionFuture future = builder.execute(); @@ -115,7 +115,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder awaitBusy(() -> ALLOWED_OPERATIONS.hasQueuedThreads() && ALLOWED_OPERATIONS.availablePermits() == 0); // Status should show the task running - TaskInfo mainTask = findTaskToCancel(action, builder.request().getSlices().number()); + TaskInfo mainTask = findTaskToCancel(action, builder.request().getSlices()); BulkByScrollTask.Status status = (BulkByScrollTask.Status) mainTask.getStatus(); assertNull(status.getReasonCancelled()); @@ -132,7 +132,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder mainTask = client().admin().cluster().prepareGetTask(mainTask.getTaskId()).get().getTask().getTask(); status = (BulkByScrollTask.Status) mainTask.getStatus(); assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled()); - if (builder.request().getSlices().number() > 1) { + if (builder.request().getSlices() > 1) { boolean foundCancelled = false; ListTasksResponse sliceList = client().admin().cluster().prepareListTasks().setParentTaskId(mainTask.getTaskId()) .setDetailed(true).get(); @@ -147,11 +147,11 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder } // Unblock the last operations - ALLOWED_OPERATIONS.release(builder.request().getSlices().number()); + ALLOWED_OPERATIONS.release(builder.request().getSlices()); // Checks that no more operations are executed assertBusy(() -> { - if (builder.request().getSlices().number() == 1) { + if (builder.request().getSlices() == 1) { /* We can only be sure that we've drained all the permits if we only use a single worker. Otherwise some worker may have * exhausted all of its documents before we blocked. */ assertEquals(0, ALLOWED_OPERATIONS.availablePermits()); @@ -172,7 +172,7 @@ private void testCancel(String action, AbstractBulkByScrollRequestBuilder assertThat(response.getBulkFailures(), emptyIterable()); assertThat(response.getSearchFailures(), emptyIterable()); - if (builder.request().getSlices().number() >= 1) { + if (builder.request().getSlices() >= 1) { // If we have more than one worker we might not have made all the modifications numModifiedDocs -= ALLOWED_OPERATIONS.availablePermits(); } @@ -232,7 +232,7 @@ public void testDeleteByQueryCancel() throws Exception { public void testReindexCancelWithWorkers() throws Exception { testCancel(ReindexAction.NAME, - reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(Slices.of(5)), + reindex().source(INDEX).filter(QueryBuilders.matchAllQuery()).destination("dest", TYPE).setSlices(5), (response, total, modified) -> { assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); refresh("dest"); @@ -250,7 +250,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { "}"); assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get()); - testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(Slices.of(5)), + testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5), (response, total, modified) -> { assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified); @@ -260,7 +260,7 @@ public void testUpdateByQueryCancelWithWorkers() throws Exception { } public void testDeleteByQueryCancelWithWorkers() throws Exception { - testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(Slices.of(5)), + testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX).filter(QueryBuilders.matchAllQuery()).setSlices(5), (response, total, modified) -> { assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")).slices(hasSize(5))); assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index 95109cd851651..276dc955f8277 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -238,9 +238,7 @@ public void testSlices() throws Exception { ); assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7); - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(2, 10)); + int slices = randomSlices(); int expectedSlices = expectedSliceStatuses(slices, "test"); // Deletes the two docs that matches "foo:a" @@ -287,9 +285,7 @@ public void testMultipleSources() throws Exception { assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); } - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(1, 10)); + int slices = randomSlices(1, 10); int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java index ccd5f08d39747..43764bf25fcbf 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java @@ -101,9 +101,7 @@ public void testCopyManyWithSlices() throws Exception { indexRandom(true, docs); assertHitCount(client().prepareSearch("source").setSize(0).get(), max); - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(2, 10)); + int slices = randomSlices(); int expectedSlices = expectedSliceStatuses(slices, "source"); // Copy all the docs @@ -144,9 +142,7 @@ public void testMultipleSources() throws Exception { assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); } - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(1, 10)); + int slices = randomSlices(1, 10); int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java index 7e32a9f3fefe6..54854afb35ea4 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java @@ -66,28 +66,40 @@ public static BulkIndexByScrollResponseMatcher matcher() { return new BulkIndexByScrollResponseMatcher(); } + static int randomSlices(int min, int max) { + if (randomBoolean()) { + return AbstractBulkByScrollRequest.AUTO_SLICES; + } else { + return between(min, max); + } + } + + static int randomSlices() { + return randomSlices(2, 10); + } + /** * Figures out how many slices the request handling will use */ - protected int expectedSlices(Slices requestSlices, Collection indices) { - if (requestSlices.isAuto()) { + protected int expectedSlices(int requestSlices, Collection indices) { + if (requestSlices == AbstractBulkByScrollRequest.AUTO_SLICES) { int leastNumShards = Collections.min(indices.stream() .map(sourceIndex -> getNumShards(sourceIndex).numPrimaries) .collect(Collectors.toList())); return Math.min(leastNumShards, BulkByScrollParallelizationHelper.AUTO_SLICE_CEILING); } else { - return requestSlices.number(); + return requestSlices; } } - protected int expectedSlices(Slices requestSlices, String index) { + protected int expectedSlices(int requestSlices, String index) { return expectedSlices(requestSlices, singleton(index)); } /** * Figures out how many slice statuses to expect in the response */ - protected int expectedSliceStatuses(Slices requestSlices, Collection indices) { + protected int expectedSliceStatuses(int requestSlices, Collection indices) { int slicesConfigured = expectedSlices(requestSlices, indices); if (slicesConfigured > 1) { @@ -97,7 +109,7 @@ protected int expectedSliceStatuses(Slices requestSlices, Collection ind } } - protected int expectedSliceStatuses(Slices slicesConfigured, String index) { + protected int expectedSliceStatuses(int slicesConfigured, String index) { return expectedSliceStatuses(slicesConfigured, singleton(index)); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java index 8eb613259dbd2..566c057a7984b 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java @@ -71,14 +71,6 @@ public void testDeleteByQueryWithWorkers() throws Exception { testCase(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).setSlices(randomSlices()), DeleteByQueryAction.NAME); } - private static Slices randomSlices() { - if (randomBoolean()) { - return Slices.AUTO; - } else { - return Slices.of(between(2, 10)); - } - } - private void testCase(AbstractBulkByScrollRequestBuilder request, String actionName) throws Exception { logger.info("Starting test for [{}] with [{}] slices", actionName, request.request().getSlices()); /* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches. diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index b2a31ef789209..946ab030c8285 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -70,17 +70,16 @@ public void testReindexRequest() throws IOException { roundTrip(reindex, tripped); assertRequestEquals(reindex, tripped); - // Try slices with a version that doesn't support slices. That should fail. - reindex.setSlices(Slices.of(between(2, 1000))); - Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, reindex, null)); - assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " - + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); + // Try slices=auto with a version that doesn't support it, which should fail + reindex.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES); + Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, reindex, null)); + assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage()); - // Try without slices with a version that doesn't support slices. That should work. + // Try regular slices with a version that doesn't support slices=auto, which should succeed tripped = new ReindexRequest(); - reindex.setSlices(Slices.of(1)); - roundTrip(Version.V_5_0_0_rc1, reindex, tripped); - assertRequestEquals(Version.V_5_0_0_rc1, reindex, tripped); + reindex.setSlices(between(1, Integer.MAX_VALUE)); + roundTrip(Version.V_6_0_0_alpha1, reindex, tripped); + assertRequestEquals(Version.V_6_0_0_alpha1, reindex, tripped); } public void testUpdateByQueryRequest() throws IOException { @@ -94,16 +93,15 @@ public void testUpdateByQueryRequest() throws IOException { assertRequestEquals(update, tripped); assertEquals(update.getPipeline(), tripped.getPipeline()); - // Try slices with a version that doesn't support slices. That should fail. - update.setSlices(Slices.of(between(2, 1000))); - Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, update, null)); - assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " - + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); + // Try slices=auto with a version that doesn't support it, which should fail + update.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES); + Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, update, null)); + assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage()); - // Try without slices with a version that doesn't support slices. That should work. + // Try regular slices with a version that doesn't support slices=auto, which should succeed tripped = new UpdateByQueryRequest(); - update.setSlices(Slices.of(1)); - roundTrip(Version.V_5_0_0_rc1, update, tripped); + update.setSlices(between(1, Integer.MAX_VALUE)); + roundTrip(Version.V_6_0_0_alpha1, update, tripped); assertRequestEquals(update, tripped); assertEquals(update.getPipeline(), tripped.getPipeline()); } @@ -115,16 +113,15 @@ public void testDeleteByQueryRequest() throws IOException { roundTrip(delete, tripped); assertRequestEquals(delete, tripped); - // Try slices with a version that doesn't support slices. That should fail. - delete.setSlices(Slices.of(between(2, 1000))); - Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_5_0_0_rc1, delete, null)); - assertEquals("Attempting to send sliced reindex-style request to a node that doesn't support it. " - + "Version is [5.0.0-rc1] but must be [5.1.1]", e.getMessage()); + // Try slices=auto with a version that doesn't support it, which should fail + delete.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES); + Exception e = expectThrows(IllegalArgumentException.class, () -> roundTrip(Version.V_6_0_0_alpha1, delete, null)); + assertEquals("Slices set as \"auto\" are not supported before version [6.1.0]. Found version [6.0.0-alpha1]", e.getMessage()); - // Try without slices with a version that doesn't support slices. That should work. + // Try regular slices with a version that doesn't support slices=auto, which should succeed tripped = new DeleteByQueryRequest(); - delete.setSlices(Slices.of(1)); - roundTrip(Version.V_5_0_0_rc1, delete, tripped); + delete.setSlices(between(1, Integer.MAX_VALUE)); + roundTrip(Version.V_6_0_0_alpha1, delete, tripped); assertRequestEquals(delete, tripped); } @@ -140,9 +137,7 @@ private void randomRequest(AbstractBulkByScrollRequest request) { request.setWaitForActiveShards(randomIntBetween(0, 10)); request.setRequestsPerSecond(between(0, Integer.MAX_VALUE)); - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(1, Integer.MAX_VALUE)); + int slices = ReindexTestCase.randomSlices(1, Integer.MAX_VALUE); request.setSlices(slices); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java index 0511eef525837..e9003b868f508 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java @@ -54,7 +54,7 @@ public class TransportRethrottleActionTests extends ESTestCase { public void createTask() { slices = between(2, 50); task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setParent(slices); + task.setSliceChildren(slices); } /** @@ -114,7 +114,7 @@ public void testRethrottleWithSomeSucceeded() { List sliceStatuses = new ArrayList<>(slices); for (int i = 0; i < succeeded; i++) { BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.getParentWorker().onSliceResponse(neverCalled(), i, + task.getSlicesParentWorker().onSliceResponse(neverCalled(), i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); sliceStatuses.add(new BulkByScrollTask.StatusOrException(status)); } @@ -135,7 +135,7 @@ public void testRethrottleWithAllSucceeded() { @SuppressWarnings("unchecked") ActionListener listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class); BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.getParentWorker().onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), + task.getSlicesParentWorker().onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); if (i == slices - 1) { // The whole thing succeeded so we should have got the success diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java index 17381ff64d293..ce254b8796911 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java @@ -81,9 +81,7 @@ public void testSlices() throws Exception { assertEquals(1, client().prepareGet("test", "test", "1").get().getVersion()); assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion()); - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(2, 10)); + int slices = randomSlices(2, 10); int expectedSlices = expectedSliceStatuses(slices, "test"); // Reindex all the docs @@ -146,9 +144,7 @@ public void testMultipleSources() throws Exception { assertHitCount(client().prepareSearch(entry.getKey()).setSize(0).get(), entry.getValue().size()); } - Slices slices = randomBoolean() - ? Slices.AUTO - : Slices.of(between(1, 10)); + int slices = randomSlices(1, 10); int expectedSlices = expectedSliceStatuses(slices, docs.keySet()); String[] sourceIndexNames = docs.keySet().toArray(new String[docs.size()]); diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml index ba837bf655eeb..715e81f5dedfc 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml @@ -136,7 +136,7 @@ --- "zero slices fails": - do: - catch: /\[slices\] must be at least 1/ + catch: /\[slices\] must be a positive integer or the string "auto"/ delete_by_query: slices: 0 index: test diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml index c6cbc647943fe..bef31b1bd799c 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml @@ -287,7 +287,7 @@ --- "zero slices fails": - do: - catch: /\[slices\] must be at least 1/ + catch: /\[slices\] must be a positive integer or the string "auto"/ reindex: slices: 0 body: diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml index 35b212c854ee6..b7499180cdaa8 100644 --- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml +++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml @@ -116,7 +116,7 @@ --- "zero slices fails": - do: - catch: /\[slices\] must be at least 1/ + catch: /\[slices\] must be a positive integer or the string "auto"/ update_by_query: slices: 0 index: test diff --git a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java index 75f03ad1075a0..1e2921cf78696 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java @@ -38,7 +38,7 @@ public abstract class AbstractAsyncBulkByScrollActionTestCase< public void setupForTest() { threadPool = new TestThreadPool(getTestName()); task = new BulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID); - task.setChild(null, Float.POSITIVE_INFINITY); + task.setSliceChild(Float.POSITIVE_INFINITY); } From 14eb56ad0a56d5051ea5874b3791b97d7641080d Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Wed, 9 Aug 2017 23:10:40 -0700 Subject: [PATCH 18/21] wip #24547 fixes for code review * Better names for the slice task strategy classes * More descriptive error messages and docs language --- .../reindex/AbstractBulkByScrollRequest.java | 21 +-- .../index/reindex/BulkByScrollTask.java | 123 ++++++++++-------- ....java => LeaderBulkByScrollTaskState.java} | 34 +++-- .../index/reindex/SuccessfullyProcessed.java | 2 +- ....java => WorkerBulkByScrollTaskState.java} | 6 +- .../java/org/elasticsearch/tasks/Task.java | 2 +- .../AbstractBulkByScrollRequestTestCase.java | 13 +- ... => LeaderBulkByScrollTaskStateTests.java} | 10 +- ... => WorkerBulkByScrollTaskStateTests.java} | 38 +++--- docs/reference/docs/delete-by-query.asciidoc | 9 +- docs/reference/docs/reindex.asciidoc | 5 +- docs/reference/docs/update-by-query.asciidoc | 9 +- .../AbstractAsyncBulkByScrollAction.java | 10 +- .../BulkByScrollParallelizationHelper.java | 64 ++++----- .../index/reindex/TransportReindexAction.java | 4 +- .../reindex/TransportRethrottleAction.java | 12 +- .../reindex/TransportUpdateByQueryAction.java | 2 +- .../reindex/AsyncBulkByScrollActionTests.java | 6 +- .../TransportRethrottleActionTests.java | 6 +- ...stractAsyncBulkByScrollActionTestCase.java | 2 +- 20 files changed, 196 insertions(+), 182 deletions(-) rename core/src/main/java/org/elasticsearch/index/reindex/{ParentBulkByScrollWorker.java => LeaderBulkByScrollTaskState.java} (80%) rename core/src/main/java/org/elasticsearch/index/reindex/{ChildBulkByScrollWorker.java => WorkerBulkByScrollTaskState.java} (97%) rename core/src/test/java/org/elasticsearch/index/reindex/{ParentBulkByScrollWorkerTests.java => LeaderBulkByScrollTaskStateTests.java} (96%) rename core/src/test/java/org/elasticsearch/index/reindex/{ChildBulkByScrollWorkerTests.java => WorkerBulkByScrollTaskStateTests.java} (89%) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index afd8181b88fc5..9a44558260ac4 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -157,7 +157,7 @@ public ActionRequestValidationException validate() { e); } if (searchRequest.source().slice() != null && slices != DEFAULT_SLICES) { - e = addValidationError("can't set a specific single slice for this request and multiple slices", e); + e = addValidationError("can't specify both manual and automatic slicing at the same time", e); } return e; } @@ -345,6 +345,9 @@ public boolean getShouldStoreResult() { * The number of slices this task should be divided into. Defaults to 1 meaning the task isn't sliced into subtasks. */ public Self setSlices(int slices) { + if (slices < 0) { + throw new IllegalArgumentException("[slices] must be at least 0"); + } this.slices = slices; return self(); } @@ -365,6 +368,10 @@ public int getSlices() { * Setup a clone of this request with the information needed to process a slice of it. */ protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) { + if (totalSlices < 1) { + throw new IllegalArgumentException("Number of total slices must be at least one"); + } + request.setAbortOnVersionConflict(abortOnVersionConflict).setRefresh(refresh).setTimeout(timeout) .setWaitForActiveShards(activeShardCount).setRetryBackoffInitialTime(retryBackoffInitialTime).setMaxRetries(maxRetries) // Parent task will store result @@ -417,15 +424,11 @@ public void writeTo(StreamOutput out) throws IOException { retryBackoffInitialTime.writeTo(out); out.writeVInt(maxRetries); out.writeFloat(requestsPerSecond); - if (out.getVersion().onOrAfter(Version.V_6_1_0)) { - out.writeVInt(slices); + if (out.getVersion().before(Version.V_6_1_0) && slices == AUTO_SLICES) { + throw new IllegalArgumentException("Slices set as \"auto\" are not supported before version [" + Version.V_6_1_0 + "]. " + + "Found version [" + out.getVersion() + "]"); } else { - if (slices == AUTO_SLICES) { - throw new IllegalArgumentException("Slices set as \"auto\" are not supported before version [" + Version.V_6_1_0 + "]. " + - "Found version [" + out.getVersion() + "]"); - } else { - out.writeVInt(slices); - } + out.writeVInt(slices); } } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java index d0526fe289877..57f7e26f75d0b 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java @@ -31,8 +31,10 @@ import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.tasks.TaskInfo; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -46,18 +48,18 @@ * * When the request is not sliced, this task is the only task created, and starts an action to perform search requests. * - * When the request is sliced, this task can either represent a parent coordinating task (using - * {@link BulkByScrollTask#setSliceChildren(int)}) or a child task that performs search queries (using - * {@link BulkByScrollTask#setSliceChild(float, Integer)}). + * When the request is sliced, this task can either represent a coordinating task (using + * {@link BulkByScrollTask#setWorkerCount(int)}) or a worker task that performs search queries (using + * {@link BulkByScrollTask#setWorker(float, Integer)}). * - * We don't always know if this task will be a parent or child task when it's created, because if slices is set to "auto" it may + * We don't always know if this task will be a leader or worker task when it's created, because if slices is set to "auto" it may * be either depending on the number of shards in the source indices. We figure that out when the request is handled and set it on this - * class with {@link #setSliceChildren(int)} or {@link #setSliceChild(float, Integer)}. + * class with {@link #setWorkerCount(int)} or {@link #setWorker(float, Integer)}. */ public class BulkByScrollTask extends CancellableTask { - private ParentBulkByScrollWorker parentWorker; - private ChildBulkByScrollWorker childWorker; + private LeaderBulkByScrollTaskState leaderState; + private WorkerBulkByScrollTaskState workerState; public BulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId) { super(id, type, action, description, parentTaskId); @@ -65,101 +67,112 @@ public BulkByScrollTask(long id, String type, String action, String description, @Override public BulkByScrollTask.Status getStatus() { - if (isSlicesParent()) { - return parentWorker.getStatus(); + if (isLeader()) { + return leaderState.getStatus(); } - if (isSliceChild()) { - return childWorker.getStatus(); + if (isWorker()) { + return workerState.getStatus(); } return emptyStatus(); } + /** + * Build the status for this task given a snapshot of the information of running slices. This is only supported if the task is + * set as a leader for slice subtasks + */ + public TaskInfo taskInfoGivenSubtaskInfo(String localNodeId, List sliceInfo) { + if (isLeader() == false) { + throw new IllegalStateException("This task is not set to be a leader of other slice subtasks"); + } + + List sliceStatuses = Arrays.asList( + new BulkByScrollTask.StatusOrException[leaderState.getSlices()]); + for (TaskInfo t : sliceInfo) { + BulkByScrollTask.Status status = (BulkByScrollTask.Status) t.getStatus(); + sliceStatuses.set(status.getSliceId(), new BulkByScrollTask.StatusOrException(status)); + } + Status status = leaderState.getStatus(sliceStatuses); + return taskInfo(localNodeId, getDescription(), status); + } + private BulkByScrollTask.Status emptyStatus() { return new Status(Collections.emptyList(), getReasonCancelled()); } /** - * Returns true if this task is a parent of other sliced tasks. False otherwise + * Returns true if this task is a leader for other slice subtasks */ - public boolean isSlicesParent() { - return parentWorker != null; + public boolean isLeader() { + return leaderState != null; } /** - * Sets this task to be a parent task for {@code slices} sliced subtasks + * Sets this task to be a leader task for {@code slices} sliced subtasks */ - public void setSliceChildren(int slices) { - if (isSlicesParent()) { - throw new IllegalStateException("Parent worker is already set"); + public void setWorkerCount(int slices) { + if (isLeader()) { + throw new IllegalStateException("This task is already a leader for other slice subtasks"); } - if (isSliceChild()) { - throw new IllegalStateException("Worker is already set as child"); + if (isWorker()) { + throw new IllegalStateException("This task is already a worker"); } - parentWorker = new ParentBulkByScrollWorker(this, slices); + leaderState = new LeaderBulkByScrollTaskState(this, slices); } /** - * Returns the worker object that tracks the state of sliced subtasks. Throws IllegalStateException if this task is not set to be - * a parent task. + * Returns the object that tracks the state of sliced subtasks. Throws IllegalStateException if this task is not set to be + * a leader task. */ - public ParentBulkByScrollWorker getSlicesParentWorker() { - if (!isSlicesParent()) { - throw new IllegalStateException("This task is not set as a parent worker"); + public LeaderBulkByScrollTaskState getLeaderState() { + if (!isLeader()) { + throw new IllegalStateException("This task is not set to be a leader for other slice subtasks"); } - return parentWorker; - } - - /** - * Returns true if this task is a child task that performs search requests. False otherwise - */ - public boolean isSliceChild() { - return childWorker != null; + return leaderState; } /** - * Sets this task to be a child task that performs search requests, when the request is not sliced. - * @param requestsPerSecond How many search requests per second this task should make + * Returns true if this task is a worker task that performs search requests. False otherwise */ - public void setSliceChild(float requestsPerSecond) { - setSliceChild(requestsPerSecond, null); + public boolean isWorker() { + return workerState != null; } /** - * Sets this task to be a slice child task that performs search requests + * Sets this task to be a worker task that performs search requests * @param requestsPerSecond How many search requests per second this task should make - * @param sliceId If this is is a sliced task, which slice number this task corresponds to + * @param sliceId If this is is a sliced task, which slice number this task corresponds to. Null if not sliced. */ - public void setSliceChild(float requestsPerSecond, Integer sliceId) { - if (isSliceChild()) { - throw new IllegalStateException("Child worker is already set"); + public void setWorker(float requestsPerSecond, @Nullable Integer sliceId) { + if (isWorker()) { + throw new IllegalStateException("This task is already a worker"); } - if (isSlicesParent()) { - throw new IllegalStateException("Worker is already set as child"); + if (isLeader()) { + throw new IllegalStateException("This task is already a leader for other slice subtasks"); } - childWorker = new ChildBulkByScrollWorker(this, sliceId, requestsPerSecond); + workerState = new WorkerBulkByScrollTaskState(this, sliceId, requestsPerSecond); } /** - * Returns the worker object that manages sending search requests. Throws IllegalStateException if this task is not set to be a - * child task. + * Returns the object that manages sending search requests. Throws IllegalStateException if this task is not set to be a + * worker task. */ - public ChildBulkByScrollWorker getSliceChildWorker() { - if (!isSliceChild()) { - throw new IllegalStateException("This task is not set as a child worker"); + public WorkerBulkByScrollTaskState getWorkerState() { + if (!isWorker()) { + throw new IllegalStateException("This task is not set to be a worker"); } - return childWorker; + return workerState; } @Override public void onCancelled() { - if (isSlicesParent()) { + if (isLeader()) { // The task cancellation task automatically finds children and cancels them, nothing extra to do - } else if (isSliceChild()) { - childWorker.handleCancel(); + } else if (isWorker()) { + workerState.handleCancel(); } else { throw new IllegalStateException("This task has not had its sliced state initialized and doesn't know how to cancel itself"); } diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskState.java similarity index 80% rename from core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java rename to core/src/main/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskState.java index 0437a525b6d0f..e7790fc07c319 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskState.java @@ -33,7 +33,7 @@ /** * Tracks the state of sliced subtasks and provides unified status information for a sliced BulkByScrollRequest. */ -public class ParentBulkByScrollWorker { +public class LeaderBulkByScrollTaskState { private final BulkByScrollTask task; @@ -47,7 +47,7 @@ public class ParentBulkByScrollWorker { */ private final AtomicInteger runningSubtasks; - public ParentBulkByScrollWorker(BulkByScrollTask task, int slices) { + public LeaderBulkByScrollTaskState(BulkByScrollTask task, int slices) { this.task = task; this.slices = slices; results = new AtomicArray<>(slices); @@ -61,34 +61,30 @@ public int getSlices() { return slices; } - public BulkByScrollTask.Status getStatus() { + /** + * Get the combined statuses of slice subtasks, merged with the given list of statuses + */ + public BulkByScrollTask.Status getStatus(List statuses) { // We only have access to the statuses of requests that have finished so we return them - List statuses = Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()]); + if (statuses.size() != results.length()) { + throw new IllegalArgumentException("Given number of statuses does not match amount of expected results"); + } addResultsToList(statuses); return new BulkByScrollTask.Status(unmodifiableList(statuses), task.getReasonCancelled()); } /** - * The number of sliced subtasks that are still running + * Get the combined statuses of sliced subtasks */ - public int runningSliceSubTasks() { - return runningSubtasks.get(); + public BulkByScrollTask.Status getStatus() { + return getStatus(Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()])); } /** - * Build the status for this task given a snapshot of the information of running slices. + * The number of sliced subtasks that are still running */ - public TaskInfo getStatusGivenSlicesTaskInfo(String localNodeId, List sliceInfo) { - /* Merge the list of finished sub requests with the provided info. If a slice is both finished and in the list then we prefer the - * finished status because we don't expect them to change after the task is finished. */ - List sliceStatuses = Arrays.asList(new BulkByScrollTask.StatusOrException[results.length()]); - for (TaskInfo t : sliceInfo) { - BulkByScrollTask.Status status = (BulkByScrollTask.Status) t.getStatus(); - sliceStatuses.set(status.getSliceId(), new BulkByScrollTask.StatusOrException(status)); - } - addResultsToList(sliceStatuses); - BulkByScrollTask.Status status = new BulkByScrollTask.Status(sliceStatuses, task.getReasonCancelled()); - return task.taskInfo(localNodeId, task.getDescription(), status); + public int runningSliceSubTasks() { + return runningSubtasks.get(); } private void addResultsToList(List sliceStatuses) { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java index 34c63c50c65a1..aab24879770af 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.reindex; /** - * Implemented by {@link ChildBulkByScrollWorker} and {@link BulkByScrollTask.Status} to consistently implement + * Implemented by {@link WorkerBulkByScrollTaskState} and {@link BulkByScrollTask.Status} to consistently implement * {@link #getSuccessfullyProcessed()}. */ public interface SuccessfullyProcessed { diff --git a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java b/core/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java similarity index 97% rename from core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java rename to core/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java index 4ff4cf1bff00e..7631eaac65fc2 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorker.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskState.java @@ -40,9 +40,9 @@ /** * Task behavior for {@link BulkByScrollTask} that does the actual work of querying and indexing */ -public class ChildBulkByScrollWorker implements SuccessfullyProcessed { +public class WorkerBulkByScrollTaskState implements SuccessfullyProcessed { - private static final Logger logger = Loggers.getLogger(ChildBulkByScrollWorker.class); + private static final Logger logger = Loggers.getLogger(WorkerBulkByScrollTaskState.class); private final BulkByScrollTask task; @@ -77,7 +77,7 @@ public class ChildBulkByScrollWorker implements SuccessfullyProcessed { */ private final AtomicReference delayedPrepareBulkRequestReference = new AtomicReference<>(); - public ChildBulkByScrollWorker(BulkByScrollTask task, Integer sliceId, float requestsPerSecond) { + public WorkerBulkByScrollTaskState(BulkByScrollTask task, Integer sliceId, float requestsPerSecond) { this.task = task; this.sliceId = sliceId; setRequestsPerSecond(requestsPerSecond); diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java index 0c0228d5e0167..bc2e8418141dc 100644 --- a/core/src/main/java/org/elasticsearch/tasks/Task.java +++ b/core/src/main/java/org/elasticsearch/tasks/Task.java @@ -89,7 +89,7 @@ public final TaskInfo taskInfo(String localNodeId, boolean detailed) { /** * Build a proper {@link TaskInfo} for this task. */ - public final TaskInfo taskInfo(String localNodeId, String description, Status status) { + protected final TaskInfo taskInfo(String localNodeId, String description, Status status) { return new TaskInfo(new TaskId(localNodeId, getId()), getType(), getAction(), description, status, startTime, System.nanoTime() - startTimeNanos, this instanceof CancellableTask, parentTask); } diff --git a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java index 1111eb44241d3..143e2416b8880 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java @@ -45,16 +45,11 @@ public void testForSlice() { original.setSize(between(0, Integer.MAX_VALUE)); } + // it's not important how many slices there are, we just need a number for forSlice int actualSlices = between(2, 1000); - if (randomBoolean()) { - original.setSlices(actualSlices); - } else { - /* - * In this case we pretend that the true number of slices will be randomSlices, since we're just verifying the request - * behavior and not the slicing itself - */ - original.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES); - } + original.setSlices(randomBoolean() + ? actualSlices + : AbstractBulkByScrollRequest.AUTO_SLICES); TaskId slicingTask = new TaskId(randomAlphaOfLength(5), randomLong()); SearchRequest sliceRequest = new SearchRequest(); diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java b/core/src/test/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskStateTests.java similarity index 96% rename from core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java rename to core/src/test/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskStateTests.java index 7c208906140aa..2b15181ca3930 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollWorkerTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/LeaderBulkByScrollTaskStateTests.java @@ -34,17 +34,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -public class ParentBulkByScrollWorkerTests extends ESTestCase { +public class LeaderBulkByScrollTaskStateTests extends ESTestCase { private int slices; private BulkByScrollTask task; - private ParentBulkByScrollWorker worker; + private LeaderBulkByScrollTaskState taskState; @Before public void createTask() { slices = between(2, 50); task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setSliceChildren(slices); - worker = task.getSlicesParentWorker(); + task.setWorkerCount(slices); + taskState = task.getLeaderState(); } public void testBasicData() { @@ -94,7 +94,7 @@ public void testProgress() { @SuppressWarnings("unchecked") ActionListener listener = slice < slices - 1 ? neverCalled() : mock(ActionListener.class); - worker.onSliceResponse(listener, slice, + taskState.onSliceResponse(listener, slice, new BulkByScrollResponse(timeValueMillis(10), sliceStatus, emptyList(), emptyList(), false)); status = task.getStatus(); diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java b/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java similarity index 89% rename from core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java rename to core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java index d156822f5da5c..23255d551358d 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ChildBulkByScrollWorkerTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java @@ -46,15 +46,15 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -public class ChildBulkByScrollWorkerTests extends ESTestCase { +public class WorkerBulkByScrollTaskStateTests extends ESTestCase { private BulkByScrollTask task; - private ChildBulkByScrollWorker worker; + private WorkerBulkByScrollTaskState workerState; @Before public void createTask() { task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setSliceChild(Float.POSITIVE_INFINITY); - worker = task.getSliceChildWorker(); + task.setWorker(Float.POSITIVE_INFINITY, null); + workerState = task.getWorkerState(); } public void testBasicData() { @@ -81,7 +81,7 @@ public void testProgress() { assertEquals(noops, status.getNoops()); long totalHits = randomIntBetween(10, 1000); - worker.setTotal(totalHits); + workerState.setTotal(totalHits); for (long p = 0; p < totalHits; p++) { status = task.getStatus(); assertEquals(totalHits, status.getTotal()); @@ -94,28 +94,28 @@ public void testProgress() { if (randomBoolean()) { created++; - worker.countCreated(); + workerState.countCreated(); } else if (randomBoolean()) { updated++; - worker.countUpdated(); + workerState.countUpdated(); } else { deleted++; - worker.countDeleted(); + workerState.countDeleted(); } if (rarely()) { versionConflicts++; - worker.countVersionConflict(); + workerState.countVersionConflict(); } if (rarely()) { batch++; - worker.countBatch(); + workerState.countBatch(); } if (rarely()) { noops++; - worker.countNoop(); + workerState.countNoop(); } } status = task.getStatus(); @@ -142,7 +142,7 @@ public void testDelayAndRethrottle() throws IOException, InterruptedException { * each time. */ float originalRequestsPerSecond = (float) randomDoubleBetween(1, 10000, true); - worker.rethrottle(originalRequestsPerSecond); + workerState.rethrottle(originalRequestsPerSecond); TimeValue maxDelay = timeValueSeconds(between(1, 5)); assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L)); int batchSizeForMaxDelay = (int) (maxDelay.seconds() * originalRequestsPerSecond); @@ -154,7 +154,7 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman } }; try { - worker.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() { + workerState.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() { @Override protected void doRun() throws Exception { boolean oldValue = done.getAndSet(true); @@ -175,7 +175,7 @@ public void onFailure(Exception e) { int rethrottles = 0; while (false == done.get()) { float requestsPerSecond = (float) randomDoubleBetween(0, originalRequestsPerSecond * 2, true); - worker.rethrottle(requestsPerSecond); + workerState.rethrottle(requestsPerSecond); rethrottles += 1; } logger.info("Rethrottled [{}] times", rethrottles); @@ -240,7 +240,7 @@ public Void get(long timeout, TimeUnit unit) throws InterruptedException, Execut }; try { // Have the task use the thread pool to delay a task that does nothing - worker.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() { + workerState.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() { @Override protected void doRun() throws Exception { } @@ -257,12 +257,12 @@ public void onFailure(Exception e) { } public void testPerfectlyThrottledBatchTime() { - worker.rethrottle(Float.POSITIVE_INFINITY); - assertThat((double) worker.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f)); + workerState.rethrottle(Float.POSITIVE_INFINITY); + assertThat((double) workerState.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f)); int total = between(0, 1000000); - worker.rethrottle(1); - assertThat((double) worker.perfectlyThrottledBatchTime(total), + workerState.rethrottle(1); + assertThat((double) workerState.perfectlyThrottledBatchTime(total), closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1))); } } diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index eeb467653e348..b2a59231d3400 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -505,8 +505,9 @@ though these are all taken at approximately the same time. [[docs-delete-by-query-picking-slices]] ===== Picking the number of slices -If slicing automatically, setting `slices` to `auto` will choose a good number -for most indices. +If slicing automatically, setting `slices` to `auto` will choose a reasonable +number for most indices. If you're slicing manually or otherwise tuning +automatic slicing, use these guidelines. Query performance is most efficient when the number of `slices` is equal to the number of shards in the index. If that number is large, (for example, @@ -514,8 +515,8 @@ number of shards in the index. If that number is large, (for example, `slices` higher than the number of shards generally does not improve efficiency and adds overhead. -Indexing performance scales linearly across available resources with the +Delete performance scales linearly across available resources with the number of slices. -Whether query or indexing performance dominates the runtime depends on the +Whether query or delete performance dominates the runtime depends on the documents being reindexed and cluster resources. diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index 9ca48155dcff6..501d8744fe663 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -926,8 +926,9 @@ though these are all taken at approximately the same time. [[docs-reindex-picking-slices]] ===== Picking the number of slices -If slicing automatically, setting `slices` to `auto` will choose a good number -for most indices. +If slicing automatically, setting `slices` to `auto` will choose a reasonable +number for most indices. If you're slicing manually or otherwise tuning +automatic slicing, use these guidelines. Query performance is most efficient when the number of `slices` is equal to the number of shards in the index. If that number is large, (for example, diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index 65fa5d5b73f47..2597fd28cb8f1 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -539,8 +539,9 @@ though these are all taken at approximately the same time. [[docs-update-by-query-picking-slices]] ===== Picking the number of slices -If slicing automatically, setting `slices` to `auto` will choose a good number -for most indices. +If slicing automatically, setting `slices` to `auto` will choose a reasonable +number for most indices. If you're slicing manually or otherwise tuning +automatic slicing, use these guidelines. Query performance is most efficient when the number of `slices` is equal to the number of shards in the index. If that number is large, (for example, @@ -548,10 +549,10 @@ number of shards in the index. If that number is large, (for example, `slices` higher than the number of shards generally does not improve efficiency and adds overhead. -Indexing performance scales linearly across available resources with the +Update performance scales linearly across available resources with the number of slices. -Whether query or indexing performance dominates the runtime depends on the +Whether query or update performance dominates the runtime depends on the documents being reindexed and cluster resources. [float] diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index d6786333b8c48..f6e7c31412894 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -88,7 +88,7 @@ public abstract class AbstractAsyncBulkByScrollAction> { protected final Logger logger; protected final BulkByScrollTask task; - protected final ChildBulkByScrollWorker worker; + protected final WorkerBulkByScrollTaskState worker; protected final ThreadPool threadPool; protected final ScriptService scriptService; protected final ClusterState clusterState; @@ -126,10 +126,10 @@ public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, Logger logger, Par ActionListener listener, Settings settings) { this.task = task; - if (!task.isSliceChild()) { + if (!task.isWorker()) { throw new IllegalArgumentException("Given task [" + task.getId() + "] must have a child worker"); } - this.worker = task.getSliceChildWorker(); + this.worker = task.getWorkerState(); this.logger = logger; this.client = client; @@ -768,7 +768,7 @@ public static RequestWrapper wrap(DeleteRequest request) { */ public abstract static class ScriptApplier implements BiFunction, ScrollableHitSource.Hit, RequestWrapper> { - private final ChildBulkByScrollWorker taskWorker; + private final WorkerBulkByScrollTaskState taskWorker; private final ScriptService scriptService; private final Script script; private final Map params; @@ -776,7 +776,7 @@ public abstract static class ScriptApplier implements BiFunction context; - public ScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, Map params) { + public ScriptApplier(WorkerBulkByScrollTaskState taskWorker, ScriptService scriptService, Script script, Map params) { this.taskWorker = taskWorker; this.scriptService = scriptService; this.script = script; diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 80f03789bc4d8..f04923116657f 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Client; @@ -51,24 +52,26 @@ private BulkByScrollParallelizationHelper() {} /** * Takes an action created by a {@link BulkByScrollTask} and runs it with regard to whether the request is sliced or not. * - * If the request is not sliced (e.g. the number of slices is 1), the action from the given {@link Supplier} will be started on the - * local node. If the request is sliced (e.g. the number of slices is more than 1), then a subrequest will be created for each slice + * If the request is not sliced (i.e. the number of slices is 1), the action from the given {@link Supplier} will be started on the + * local node. If the request is sliced (i.e. the number of slices is more than 1), then a subrequest will be created for each slice * and sent. * * If slices are set as {@code "auto"}, this class will resolve that to a specific number based on characteristics of the source * indices. A request with {@code "auto"} slices may end up being sliced or unsliced. */ static > void startSlicedAction( - Request request, - BulkByScrollTask task, - Action action, - ActionListener listener, - Client client, - DiscoveryNode node, - Supplier> taskSupplier) { + Request request, + BulkByScrollTask task, + Action action, + ActionListener listener, + Client client, + DiscoveryNode node, + Supplier> taskSupplier) { if (request.getSlices() == AbstractBulkByScrollRequest.AUTO_SLICES) { - client.admin().cluster().prepareSearchShards(request.getSearchRequest().indices()).execute(ActionListener.wrap( + ClusterSearchShardsRequest shardsRequest = new ClusterSearchShardsRequest(); + shardsRequest.indices(request.getSearchRequest().indices()); + client.admin().cluster().searchShards(shardsRequest, ActionListener.wrap( response -> { int actualNumSlices = countSlicesBasedOnShards(response); sliceConditionally(request, task, action, listener, client, node, taskSupplier, actualNumSlices); @@ -81,23 +84,24 @@ static > void startSlicedAc } private static > void sliceConditionally( - Request request, - BulkByScrollTask task, - Action action, - ActionListener listener, - Client client, - DiscoveryNode node, - Supplier> taskSupplier, - int slices) { + Request request, + BulkByScrollTask task, + Action action, + ActionListener listener, + Client client, + DiscoveryNode node, + Supplier> taskSupplier, + int slices) { if (slices > 1) { - task.setSliceChildren(slices); + task.setWorkerCount(slices); sendSubRequests(client, action, node.getId(), task, request, listener); } else { - Integer sliceId = request.getSearchRequest().source().slice() == null + SliceBuilder sliceBuilder = request.getSearchRequest().source().slice(); + Integer sliceId = sliceBuilder == null ? null - : request.getSearchRequest().source().slice().getId(); - task.setSliceChild(request.getRequestsPerSecond(), sliceId); + : sliceBuilder.getId(); + task.setWorker(request.getRequestsPerSecond(), sliceId); taskSupplier.get().start(); } } @@ -105,7 +109,7 @@ private static > void slice private static int countSlicesBasedOnShards(ClusterSearchShardsResponse response) { Map countsByIndex = Arrays.stream(response.getGroups()).collect(Collectors.toMap( group -> group.getShardId().getIndex(), - __ -> 1, + group -> 1, (sum, term) -> sum + term )); Set counts = new HashSet<>(countsByIndex.values()); @@ -114,14 +118,14 @@ private static int countSlicesBasedOnShards(ClusterSearchShardsResponse response } private static > void sendSubRequests( - Client client, - Action action, - String localNodeId, - BulkByScrollTask task, - Request request, - ActionListener listener) { + Client client, + Action action, + String localNodeId, + BulkByScrollTask task, + Request request, + ActionListener listener) { - ParentBulkByScrollWorker worker = task.getSlicesParentWorker(); + LeaderBulkByScrollTaskState worker = task.getLeaderState(); int totalSlices = worker.getSlices(); TaskId parentTaskId = new TaskId(localNodeId, task.getId()); for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, totalSlices)) { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index e2c3c7163fddd..70e4f8009d7d6 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -389,8 +389,8 @@ protected void copyRouting(RequestWrapper request, String routing) { class ReindexScriptApplier extends ScriptApplier { - ReindexScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, - Map params) { + ReindexScriptApplier(WorkerBulkByScrollTaskState taskWorker, ScriptService scriptService, Script script, + Map params) { super(taskWorker, scriptService, script, params); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java index 444b2c9d1040c..d8105e4a6ec86 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java @@ -60,12 +60,12 @@ protected void taskOperation(RethrottleRequest request, BulkByScrollTask task, A static void rethrottle(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { - if (task.isSliceChild()) { + if (task.isWorker()) { rethrottleChildTask(logger, localNodeId, task, newRequestsPerSecond, listener); return; } - if (task.isSlicesParent()) { + if (task.isLeader()) { rethrottleParentTask(logger, localNodeId, client, task, newRequestsPerSecond, listener); return; } @@ -75,8 +75,8 @@ static void rethrottle(Logger logger, String localNodeId, Client client, BulkByS private static void rethrottleParentTask(Logger logger, String localNodeId, Client client, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { - final ParentBulkByScrollWorker parentWorker = task.getSlicesParentWorker(); - final int runningSubtasks = parentWorker.runningSliceSubTasks(); + final LeaderBulkByScrollTaskState leaderState = task.getLeaderState(); + final int runningSubtasks = leaderState.runningSliceSubTasks(); if (runningSubtasks > 0) { RethrottleRequest subRequest = new RethrottleRequest(); @@ -87,7 +87,7 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie client.execute(RethrottleAction.INSTANCE, subRequest, ActionListener.wrap( r -> { r.rethrowFailures("Rethrottle"); - listener.onResponse(parentWorker.getStatusGivenSlicesTaskInfo(localNodeId, r.getTasks())); + listener.onResponse(task.taskInfoGivenSubtaskInfo(localNodeId, r.getTasks())); }, listener::onFailure)); } else { @@ -99,7 +99,7 @@ private static void rethrottleParentTask(Logger logger, String localNodeId, Clie private static void rethrottleChildTask(Logger logger, String localNodeId, BulkByScrollTask task, float newRequestsPerSecond, ActionListener listener) { logger.debug("rethrottling local task [{}] to [{}] requests per second", task.getId(), newRequestsPerSecond); - task.getSliceChildWorker().rethrottle(newRequestsPerSecond); + task.getWorkerState().rethrottle(newRequestsPerSecond); listener.onResponse(task.taskInfo(localNodeId, true)); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index fd4568c27d47d..3068c57654c7a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -131,7 +131,7 @@ protected RequestWrapper buildRequest(ScrollableHitSource.Hit doc) class UpdateByQueryScriptApplier extends ScriptApplier { - UpdateByQueryScriptApplier(ChildBulkByScrollWorker taskWorker, ScriptService scriptService, Script script, + UpdateByQueryScriptApplier(WorkerBulkByScrollTaskState taskWorker, ScriptService scriptService, Script script, Map params) { super(taskWorker, scriptService, script, params); } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index 62f2c02f01162..fe754b3881727 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -125,7 +125,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase { private String scrollId; private TaskManager taskManager; private BulkByScrollTask testTask; - private ChildBulkByScrollWorker worker; + private WorkerBulkByScrollTaskState worker; private Map expectedHeaders = new HashMap<>(); private DiscoveryNode localNode; private TaskId taskId; @@ -143,8 +143,8 @@ public void setupForTest() { scrollId = null; taskManager = new TaskManager(Settings.EMPTY); testTask = (BulkByScrollTask) taskManager.register("don'tcare", "hereeither", testRequest); - testTask.setSliceChild(testRequest.getRequestsPerSecond()); - worker = testTask.getSliceChildWorker(); + testTask.setWorker(testRequest.getRequestsPerSecond(), null); + worker = testTask.getWorkerState(); localNode = new DiscoveryNode("thenode", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); taskId = new TaskId(localNode.getId(), testTask.getId()); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java index e9003b868f508..62a2c34ea582c 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java @@ -54,7 +54,7 @@ public class TransportRethrottleActionTests extends ESTestCase { public void createTask() { slices = between(2, 50); task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID); - task.setSliceChildren(slices); + task.setWorkerCount(slices); } /** @@ -114,7 +114,7 @@ public void testRethrottleWithSomeSucceeded() { List sliceStatuses = new ArrayList<>(slices); for (int i = 0; i < succeeded; i++) { BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.getSlicesParentWorker().onSliceResponse(neverCalled(), i, + task.getLeaderState().onSliceResponse(neverCalled(), i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); sliceStatuses.add(new BulkByScrollTask.StatusOrException(status)); } @@ -135,7 +135,7 @@ public void testRethrottleWithAllSucceeded() { @SuppressWarnings("unchecked") ActionListener listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class); BulkByScrollTask.Status status = believeableCompletedStatus(i); - task.getSlicesParentWorker().onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), + task.getLeaderState().onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false)); if (i == slices - 1) { // The whole thing succeeded so we should have got the success diff --git a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java index 1e2921cf78696..a0752b0048564 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionTestCase.java @@ -38,7 +38,7 @@ public abstract class AbstractAsyncBulkByScrollActionTestCase< public void setupForTest() { threadPool = new TestThreadPool(getTestName()); task = new BulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID); - task.setSliceChild(Float.POSITIVE_INFINITY); + task.setWorker(Float.POSITIVE_INFINITY, null); } From 5500bcf0ecfc09ab14a2b0145b42e2fdf4d5e49e Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Wed, 9 Aug 2017 23:23:09 -0700 Subject: [PATCH 19/21] wip #24547 checkstyle fixes --- .../WorkerBulkByScrollTaskStateTests.java | 24 ++++++++++--------- .../AbstractAsyncBulkByScrollAction.java | 5 +++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java b/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java index 23255d551358d..64bf52c319e68 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/WorkerBulkByScrollTaskStateTests.java @@ -154,20 +154,22 @@ public ScheduledFuture schedule(TimeValue delay, String name, Runnable comman } }; try { - workerState.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - boolean oldValue = done.getAndSet(true); - if (oldValue) { - throw new RuntimeException("Ran twice oh no!"); + workerState.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, + new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + boolean oldValue = done.getAndSet(true); + if (oldValue) { + throw new RuntimeException("Ran twice oh no!"); + } } - } - @Override - public void onFailure(Exception e) { - errors.add(e); + @Override + public void onFailure(Exception e) { + errors.add(e); + } } - }); + ); // Rethrottle on a random number of threads, one of which is this thread. Runnable test = () -> { diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index f6e7c31412894..0fc89677f4025 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -776,7 +776,10 @@ public abstract static class ScriptApplier implements BiFunction context; - public ScriptApplier(WorkerBulkByScrollTaskState taskWorker, ScriptService scriptService, Script script, Map params) { + public ScriptApplier(WorkerBulkByScrollTaskState taskWorker, + ScriptService scriptService, + Script script, + Map params) { this.taskWorker = taskWorker; this.scriptService = scriptService; this.script = script; From eeffd397fbc72041469ee4a36b286b93d52b8e83 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 10 Aug 2017 09:05:23 -0700 Subject: [PATCH 20/21] wip #24547 fix broken test --- .../org/elasticsearch/index/reindex/ReindexRequestTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java index 8a7c7323348b0..9f4b20ff35ba3 100644 --- a/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java @@ -62,7 +62,7 @@ public void testNoSliceBuilderSetWithSlicedRequest() { reindex.getSearchRequest().source().slice(new SliceBuilder(0, 4)); reindex.setSlices(between(2, Integer.MAX_VALUE)); ActionRequestValidationException e = reindex.validate(); - assertEquals("Validation Failed: 1: can't set a specific single slice for this request and multiple slices;", e.getMessage()); + assertEquals("Validation Failed: 1: can't specify both manual and automatic slicing at the same time;", e.getMessage()); } @Override From f5b5c186237905b5499d757ea7e8eb186cdc2ffc Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 10 Aug 2017 14:46:18 -0700 Subject: [PATCH 21/21] wip #24547 code review fixes --- .../reindex/AbstractBulkByScrollRequest.java | 4 ++-- .../reindex/AbstractBaseReindexRestHandler.java | 6 ++++-- .../BulkByScrollParallelizationHelper.java | 17 ++++++++--------- .../reindex/TransportDeleteByQueryAction.java | 4 ++-- .../index/reindex/TransportReindexAction.java | 4 ++-- .../reindex/TransportUpdateByQueryAction.java | 4 ++-- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java index 9a44558260ac4..0355eeaee4788 100644 --- a/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java +++ b/core/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java @@ -346,7 +346,7 @@ public boolean getShouldStoreResult() { */ public Self setSlices(int slices) { if (slices < 0) { - throw new IllegalArgumentException("[slices] must be at least 0"); + throw new IllegalArgumentException("[slices] must be at least 0 but was [" + slices + "]"); } this.slices = slices; return self(); @@ -369,7 +369,7 @@ public int getSlices() { */ protected Self doForSlice(Self request, TaskId slicingTask, int totalSlices) { if (totalSlices < 1) { - throw new IllegalArgumentException("Number of total slices must be at least one"); + throw new IllegalArgumentException("Number of total slices must be at least 1 but was [" + totalSlices + "]"); } request.setAbortOnVersionConflict(abortOnVersionConflict).setRefresh(refresh).setTimeout(timeout) diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java index edf1e610ac780..4ea2592801db8 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java @@ -134,11 +134,13 @@ private static Integer parseSlices(RestRequest request) { try { slices = Integer.parseInt(slicesString); } catch (NumberFormatException e) { - throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\"", e); + throw new IllegalArgumentException( + "[slices] must be a positive integer or the string \"auto\", but was [" + slicesString + "]", e); } if (slices < 1) { - throw new IllegalArgumentException("[slices] must be a positive integer or the string \"auto\""); + throw new IllegalArgumentException( + "[slices] must be a positive integer or the string \"auto\", but was [" + slicesString + "]"); } return slices; diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index f04923116657f..19cca917290f9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -37,7 +37,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -52,9 +51,9 @@ private BulkByScrollParallelizationHelper() {} /** * Takes an action created by a {@link BulkByScrollTask} and runs it with regard to whether the request is sliced or not. * - * If the request is not sliced (i.e. the number of slices is 1), the action from the given {@link Supplier} will be started on the - * local node. If the request is sliced (i.e. the number of slices is more than 1), then a subrequest will be created for each slice - * and sent. + * If the request is not sliced (i.e. the number of slices is 1), the worker action in the given {@link Runnable} will be started on + * the local node. If the request is sliced (i.e. the number of slices is more than 1), then a subrequest will be created for each + * slice and sent. * * If slices are set as {@code "auto"}, this class will resolve that to a specific number based on characteristics of the source * indices. A request with {@code "auto"} slices may end up being sliced or unsliced. @@ -66,7 +65,7 @@ static > void startSlicedAc ActionListener listener, Client client, DiscoveryNode node, - Supplier> taskSupplier) { + Runnable workerAction) { if (request.getSlices() == AbstractBulkByScrollRequest.AUTO_SLICES) { ClusterSearchShardsRequest shardsRequest = new ClusterSearchShardsRequest(); @@ -74,12 +73,12 @@ static > void startSlicedAc client.admin().cluster().searchShards(shardsRequest, ActionListener.wrap( response -> { int actualNumSlices = countSlicesBasedOnShards(response); - sliceConditionally(request, task, action, listener, client, node, taskSupplier, actualNumSlices); + sliceConditionally(request, task, action, listener, client, node, workerAction, actualNumSlices); }, listener::onFailure )); } else { - sliceConditionally(request, task, action, listener, client, node, taskSupplier, request.getSlices()); + sliceConditionally(request, task, action, listener, client, node, workerAction, request.getSlices()); } } @@ -90,7 +89,7 @@ private static > void slice ActionListener listener, Client client, DiscoveryNode node, - Supplier> taskSupplier, + Runnable workerAction, int slices) { if (slices > 1) { @@ -102,7 +101,7 @@ private static > void slice ? null : sliceBuilder.getId(); task.setWorker(request.getRequestsPerSecond(), sliceId); - taskSupplier.get().start(); + workerAction.run(); } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java index 719dc34efd9e7..e2de5cd4ffc55 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java @@ -58,8 +58,8 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener { ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - return new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, - listener); + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + listener).start(); } ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java index 3068c57654c7a..e21a6408bd85a 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java @@ -71,8 +71,8 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener ClusterState state = clusterService.state(); ParentTaskAssigningClient assigningClient = new ParentTaskAssigningClient(client, clusterService.localNode(), bulkByScrollTask); - return new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, - listener); + new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, request, scriptService, state, + listener).start(); } ); }