From e0d6846920b9039c50775dcb605387bab1f7c6a8 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 17 Jan 2019 19:58:55 +0200 Subject: [PATCH 01/12] [ML] Data frame analytics as persistent tasks --- .../xpack/core/XPackClientPlugin.java | 5 +- .../elasticsearch/xpack/core/ml/MlTasks.java | 27 ++ .../action/AbstractGetResourcesRequest.java | 75 +++++ .../action/AbstractGetResourcesResponse.java | 82 +++++ .../action/GetDataFrameAnalyticsAction.java | 59 ++++ .../GetDataFrameAnalyticsStatsAction.java | 279 ++++++++++++++++++ .../action/PutDataFrameAnalyticsAction.java | 107 +++++++ .../core/ml/action/RunAnalyticsAction.java | 2 +- .../action/StartDataFrameAnalyticsAction.java | 165 +++++++++++ .../ml/dataframe/DataFrameAnalysisConfig.java | 66 +++++ .../dataframe/DataFrameAnalyticsConfig.java | 163 ++++++++++ .../ml/dataframe/DataFrameAnalyticsState.java | 36 +++ .../DataFrameAnalyticsTaskState.java | 97 ++++++ .../persistence/ElasticsearchMappings.java | 28 ++ .../core/ml/process/writer/RecordWriter.java | 2 +- .../xpack/core/ml/utils/ExceptionsHelper.java | 5 + .../DataFrameAnalysisConfigTests.java | 47 +++ .../DataFrameAnalyticsConfigTests.java | 42 +++ .../xpack/ml/MachineLearning.java | 49 ++- .../AbstractTransportGetResourcesAction.java | 126 ++++++++ .../TransportGetDataFrameAnalyticsAction.java | 65 ++++ ...sportGetDataFrameAnalyticsStatsAction.java | 99 +++++++ .../TransportPutDataFrameAnalyticsAction.java | 74 +++++ ...ransportStartDataFrameAnalyticsAction.java | 187 ++++++++++++ .../xpack/ml/dataframe/DataFrameAnalysis.java | 32 -- ...lds.java => DataFrameAnalyticsFields.java} | 4 +- .../DataFrameAnalyticsManager.java} | 182 ++++++------ .../analyses/AbstractDataFrameAnalysis.java | 38 +++ .../analyses/DataFrameAnalysesUtils.java | 83 ++++++ .../dataframe/analyses/DataFrameAnalysis.java | 42 +++ .../dataframe/analyses/OutlierDetection.java | 64 ++++ .../DataFrameDataExtractor.java | 5 +- .../DataFrameDataExtractorContext.java | 2 +- .../DataFrameDataExtractorFactory.java | 4 +- .../DataFrameAnalyticsConfigProvider.java | 83 ++++++ .../dataframe/process/AnalyticsProcess.java | 2 +- .../process/AnalyticsProcessConfig.java | 2 +- .../process/AnalyticsProcessManager.java | 37 ++- .../process/AnalyticsResultProcessor.java | 6 +- .../NativeAnalyticsProcessFactory.java | 4 +- .../writer/AbstractDataToProcessWriter.java | 4 +- .../RestGetDataFrameAnalyticsAction.java | 50 ++++ .../RestGetDataFrameAnalyticsStatsAction.java | 50 ++++ .../RestPutDataFrameAnalyticsAction.java | 43 +++ .../RestStartDataFrameAnalyticsAction.java} | 22 +- .../DataFrameDataExtractorFactoryTests.java | 2 +- .../AnalyticsResultProcessorTests.java | 2 +- 47 files changed, 2467 insertions(+), 183 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfigTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalysis.java rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/{DataFrameFields.java => DataFrameAnalyticsFields.java} (79%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/{action/TransportRunAnalyticsAction.java => dataframe/DataFrameAnalyticsManager.java} (50%) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysis.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetection.java rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/{ => extractor}/DataFrameDataExtractor.java (97%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/{ => extractor}/DataFrameDataExtractorContext.java (95%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/{ => extractor}/DataFrameDataExtractorFactory.java (98%) create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/{analytics/RestRunAnalyticsAction.java => dataframe/RestStartDataFrameAnalyticsAction.java} (50%) rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/{ => extractor}/DataFrameDataExtractorFactoryTests.java (99%) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 301cda08d5dfb..77ac71c0cf94d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -53,9 +53,9 @@ import org.elasticsearch.xpack.core.indexlifecycle.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleAction; import org.elasticsearch.xpack.core.indexlifecycle.LifecycleType; -import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ReadOnlyAction; import org.elasticsearch.xpack.core.indexlifecycle.RolloverAction; +import org.elasticsearch.xpack.core.indexlifecycle.SetPriorityAction; import org.elasticsearch.xpack.core.indexlifecycle.ShrinkAction; import org.elasticsearch.xpack.core.indexlifecycle.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.indexlifecycle.UnfollowAction; @@ -111,6 +111,7 @@ import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; import org.elasticsearch.xpack.core.ml.action.RunAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; +import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction; @@ -450,6 +451,8 @@ public List getNamedXContent() { StartDatafeedAction.DatafeedParams::fromXContent), new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.JOB_TASK_NAME), OpenJobAction.JobParams::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME), + StartDataFrameAnalyticsAction.TaskParams::fromXContent), // ML - Task states new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(DatafeedState.NAME), DatafeedState::fromXContent), new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(JobTaskState.NAME), JobTaskState::fromXContent), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java index cd32505a48e3e..649a77648eafb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlTasks.java @@ -11,6 +11,8 @@ import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; @@ -23,9 +25,11 @@ public final class MlTasks { public static final String JOB_TASK_NAME = "xpack/ml/job"; public static final String DATAFEED_TASK_NAME = "xpack/ml/datafeed"; + public static final String DATA_FRAME_ANALYTICS_TASK_NAME = "xpack/ml/data_frame/analytics"; public static final String JOB_TASK_ID_PREFIX = "job-"; public static final String DATAFEED_TASK_ID_PREFIX = "datafeed-"; + private static final String DATA_FRAME_ANALYTICS_TASK_ID_PREFIX = "data_frame_analytics-"; public static final PersistentTasksCustomMetaData.Assignment AWAITING_UPGRADE = new PersistentTasksCustomMetaData.Assignment(null, @@ -50,6 +54,13 @@ public static String datafeedTaskId(String datafeedId) { return DATAFEED_TASK_ID_PREFIX + datafeedId; } + /** + * Namespaces the task ids for data frame analytics. + */ + public static String dataFrameAnalyticsTaskId(String id) { + return DATA_FRAME_ANALYTICS_TASK_ID_PREFIX + id; + } + @Nullable public static PersistentTasksCustomMetaData.PersistentTask getJobTask(String jobId, @Nullable PersistentTasksCustomMetaData tasks) { return tasks == null ? null : tasks.getTask(jobTaskId(jobId)); @@ -61,6 +72,12 @@ public static PersistentTasksCustomMetaData.PersistentTask getDatafeedTask(St return tasks == null ? null : tasks.getTask(datafeedTaskId(datafeedId)); } + @Nullable + public static PersistentTasksCustomMetaData.PersistentTask getDataFrameAnalyticsTask(String analyticsId, + @Nullable PersistentTasksCustomMetaData tasks) { + return tasks == null ? null : tasks.getTask(dataFrameAnalyticsTaskId(analyticsId)); + } + /** * Note that the return value of this method does NOT take node relocations into account. * Use {@link #getJobStateModifiedForReassignments} to return a value adjusted to the most @@ -120,6 +137,16 @@ public static DatafeedState getDatafeedState(String datafeedId, @Nullable Persis } } + public static DataFrameAnalyticsState getDataFrameAnalyticsState(String analyticsId, @Nullable PersistentTasksCustomMetaData tasks) { + PersistentTasksCustomMetaData.PersistentTask task = getDataFrameAnalyticsTask(analyticsId, tasks); + if (task != null && task.getState() != null) { + DataFrameAnalyticsTaskState taskState = (DataFrameAnalyticsTaskState) task.getState(); + return taskState.getState(); + } else { + return DataFrameAnalyticsState.STOPPED; + } + } + /** * The job Ids of anomaly detector job tasks. * All anomaly detector jobs are returned regardless of the status of the diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesRequest.java new file mode 100644 index 0000000000000..baa7d2714ec8b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesRequest.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; + +import java.io.IOException; +import java.util.Objects; + +public abstract class AbstractGetResourcesRequest extends ActionRequest { + + private String resourceId; + private PageParams pageParams = PageParams.defaultParams(); + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getResourceId() { + return resourceId; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + public PageParams getPageParams() { + return pageParams; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + resourceId = in.readOptionalString(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(resourceId); + out.writeOptionalWriteable(pageParams); + } + + @Override + public int hashCode() { + return Objects.hash(resourceId, pageParams); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj instanceof AbstractGetResourcesRequest == false) { + return false; + } + AbstractGetResourcesRequest other = (AbstractGetResourcesRequest) obj; + return Objects.equals(resourceId, other.resourceId); + } + + public abstract String getResourceIdField(); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java new file mode 100644 index 0000000000000..f65d4d99c6697 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; + +import java.io.IOException; +import java.util.Objects; + +public abstract class AbstractGetResourcesResponse extends ActionResponse + implements StatusToXContentObject { + + private QueryPage resources; + + protected AbstractGetResourcesResponse(QueryPage resources) { + this.resources = Objects.requireNonNull(resources); + } + + public QueryPage getResources() { + return resources; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + resources = new QueryPage<>(in, getReader()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + resources.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + resources.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(resources); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj instanceof AbstractGetResourcesResponse == false) { + return false; + } + AbstractGetResourcesResponse other = (AbstractGetResourcesResponse) obj; + return Objects.equals(resources, other.resources); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + protected abstract Reader getReader(); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..0703e17f86e9b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; + +import java.util.Collections; + +public class GetDataFrameAnalyticsAction extends Action { + + public static final GetDataFrameAnalyticsAction INSTANCE = new GetDataFrameAnalyticsAction(); + public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/get"; + + private GetDataFrameAnalyticsAction() { + super(NAME); + } + + @Override + public Response newResponse() { + return new Response(new QueryPage<>(Collections.emptyList(), 0, Response.CONFIGS)); + } + + public static class Request extends AbstractGetResourcesRequest { + + @Override + public String getResourceIdField() { + return DataFrameAnalyticsConfig.ID.getPreferredName(); + } + } + + public static class Response extends AbstractGetResourcesResponse { + + public static final ParseField CONFIGS = new ParseField("configs"); + + public Response(QueryPage analytics) { + super(analytics); + } + + @Override + protected Reader getReader() { + return DataFrameAnalyticsConfig::new; + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java new file mode 100644 index 0000000000000..e7824bb08fb7a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsAction.java @@ -0,0 +1,279 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class GetDataFrameAnalyticsStatsAction extends Action { + + public static final GetDataFrameAnalyticsStatsAction INSTANCE = new GetDataFrameAnalyticsStatsAction(); + public static final String NAME = "cluster:monitor/xpack/ml/data_frame/analytics/stats/get"; + + private GetDataFrameAnalyticsStatsAction() { + super(NAME); + } + + @Override + public Response newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return Response::new; + } + + public static class Request extends MasterNodeRequest { + + private String id; + private PageParams pageParams = PageParams.defaultParams(); + + public Request(String id) { + this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID.getPreferredName()); + } + + public Request() {} + + public Request(StreamInput in) throws IOException { + super(in); + id = in.readString(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + out.writeOptionalWriteable(pageParams); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + public PageParams getPageParams() { + return pageParams; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public int hashCode() { + return Objects.hash(id, pageParams); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(id, other.id) && Objects.equals(pageParams, other.pageParams); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetDataFrameAnalyticsStatsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + public static class Stats implements ToXContentObject, Writeable { + + private final String id; + private final DataFrameAnalyticsState state; + @Nullable + private final DiscoveryNode node; + @Nullable + private final String assignmentExplanation; + + public Stats(String id, DataFrameAnalyticsState state, @Nullable DiscoveryNode node, + @Nullable String assignmentExplanation) { + this.id = Objects.requireNonNull(id); + this.state = Objects.requireNonNull(state); + this.node = node; + this.assignmentExplanation = assignmentExplanation; + } + + public Stats(StreamInput in) throws IOException { + id = in.readString(); + state = DataFrameAnalyticsState.fromStream(in); + node = in.readOptionalWriteable(DiscoveryNode::new); + assignmentExplanation = in.readOptionalString(); + } + + public String getId() { + return id; + } + + public DataFrameAnalyticsState getState() { + return state; + } + + public DiscoveryNode getNode() { + return node; + } + + public String getAssignmentExplanation() { + return assignmentExplanation; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // TODO: Have callers wrap the content with an object as they choose rather than forcing it upon them + builder.startObject(); + { + toUnwrappedXContent(builder); + } + return builder.endObject(); + } + + public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOException { + builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id); + builder.field("state", state.toString()); + if (node != null) { + builder.startObject("node"); + builder.field("id", node.getId()); + builder.field("name", node.getName()); + builder.field("ephemeral_id", node.getEphemeralId()); + builder.field("transport_address", node.getAddress().toString()); + + builder.startObject("attributes"); + for (Map.Entry entry : node.getAttributes().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + builder.endObject(); + } + if (assignmentExplanation != null) { + builder.field("assignment_explanation", assignmentExplanation); + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + state.writeTo(out); + out.writeOptionalWriteable(node); + out.writeOptionalString(assignmentExplanation); + } + + @Override + public int hashCode() { + return Objects.hash(id, state, node, assignmentExplanation); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Stats other = (Stats) obj; + return Objects.equals(id, other.id) + && Objects.equals(this.state, other.state) + && Objects.equals(this.node, other.node) + && Objects.equals(this.assignmentExplanation, other.assignmentExplanation); + } + } + + private QueryPage stats; + + public Response() {} + + public Response(QueryPage stats) { + this.stats = stats; + } + + public Response(StreamInput in) throws IOException { + super(in); + stats = new QueryPage<>(in, Stats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + stats.writeTo(out); + } + + public QueryPage getResponse() { + return stats; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + stats.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(stats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(stats, other.stats); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..ec21024783e1e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.Objects; + +public class PutDataFrameAnalyticsAction extends Action { + + public static final PutDataFrameAnalyticsAction INSTANCE = new PutDataFrameAnalyticsAction(); + public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/put"; + + private PutDataFrameAnalyticsAction() { + super(NAME); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContentObject { + + public static Request parseRequest(String id, XContentParser parser) { + DataFrameAnalyticsConfig.Builder config = DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null); + if (config.getId() == null) { + config.setId(id); + } else if (!Strings.isNullOrEmpty(id) && !id.equals(config.getId())) { + // If we have both URI and body ID, they must be identical + throw new IllegalArgumentException(Messages.getMessage(Messages.INCONSISTENT_ID, DataFrameAnalyticsConfig.ID, + config.getId(), id)); + } + + return new PutDataFrameAnalyticsAction.Request(config.build()); + } + + private DataFrameAnalyticsConfig config; + + public Request() {} + + public Request(DataFrameAnalyticsConfig config) { + this.config = config; + } + + public DataFrameAnalyticsConfig getConfig() { + return config; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + config.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PutDataFrameAnalyticsAction.Request request = (PutDataFrameAnalyticsAction.Request) o; + return Objects.equals(config, request.config); + } + + @Override + public int hashCode() { + return Objects.hash(config); + } + } + + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, PutDataFrameAnalyticsAction action) { + super(client, action, new Request()); + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java index 0e2a4eb15eb04..5d25e68e005f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java @@ -23,7 +23,7 @@ public class RunAnalyticsAction extends Action { public static final RunAnalyticsAction INSTANCE = new RunAnalyticsAction(); - public static final String NAME = "cluster:admin/xpack/ml/analytics/run"; + public static final String NAME = "cluster:admin/xpack/ml/dataframe/run"; private RunAnalyticsAction() { super(NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..2a3f7fbd008a2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.Version; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class StartDataFrameAnalyticsAction extends Action { + + public static final StartDataFrameAnalyticsAction INSTANCE = new StartDataFrameAnalyticsAction(); + public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/start"; + + private StartDataFrameAnalyticsAction() { + super(NAME); + } + + @Override + public AcknowledgedResponse newResponse() { + return new AcknowledgedResponse(); + } + + public static class Request extends MasterNodeRequest implements ToXContentObject { + + private String id; + + public Request(String id) { + this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID); + } + + public Request(StreamInput in) throws IOException { + readFrom(in); + } + + public Request() { + } + + public String getId() { + return id; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (id != null) { + builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id); + } + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + StartDataFrameAnalyticsAction.Request other = (StartDataFrameAnalyticsAction.Request) obj; + return Objects.equals(id, other.id); + } + + @Override + public String toString() { + return Strings.toString(this); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, StartDataFrameAnalyticsAction action) { + super(client, action, new Request()); + } + } + + public static class TaskParams implements XPackPlugin.XPackPersistentTaskParams { + + public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, true, a -> new TaskParams((String) a[0])); + + public static TaskParams fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private String id; + + public TaskParams(String id) { + this.id = Objects.requireNonNull(id); + } + + public TaskParams(StreamInput in) throws IOException { + this.id = in.readString(); + } + + public String getId() { + return id; + } + + @Override + public String getWriteableName() { + return MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME; + } + + @Override + public Version getMinimalSupportedVersion() { + // TODO Update to first released version + return Version.CURRENT; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id); + builder.endObject(); + return builder; + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java new file mode 100644 index 0000000000000..9c7ce7a9ce31d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class DataFrameAnalysisConfig implements ToXContentObject, Writeable { + + public static ContextParser parser() { + return (p, c) -> new DataFrameAnalysisConfig(p.map()); + } + + private final Map config; + + public DataFrameAnalysisConfig(Map config) { + if (config.size() != 1) { + throw ExceptionsHelper.badRequestException("A data frame analysis must specify exactly one analysis type"); + } + this.config = Objects.requireNonNull(config); + } + + public DataFrameAnalysisConfig(StreamInput in) throws IOException { + config = in.readMap(); + } + + public Map asMap() { + return config; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(config); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.map(config); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataFrameAnalysisConfig that = (DataFrameAnalysisConfig) o; + return Objects.equals(config, that.config); + } + + @Override + public int hashCode() { + return Objects.hash(config); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java new file mode 100644 index 0000000000000..9e32d444b58fe --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { + + public static final String TYPE = "data_frame_analytics_config"; + + public static final ParseField ID = new ParseField("id"); + public static final ParseField SOURCE = new ParseField("source"); + public static final ParseField DEST = new ParseField("dest"); + public static final ParseField ANALYSES = new ParseField("analyses"); + public static final ParseField CONFIG_TYPE = new ParseField("config_type"); + + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); + + public static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>(TYPE, ignoreUnknownFields, Builder::new); + + parser.declareString((c, s) -> {}, CONFIG_TYPE); + parser.declareString(Builder::setId, ID); + parser.declareString(Builder::setSource, SOURCE); + parser.declareString(Builder::setDest, DEST); + parser.declareObjectArray(Builder::setAnalyses, DataFrameAnalysisConfig.parser(), ANALYSES); + return parser; + } + + private final String id; + private final String source; + private final String dest; + private final List analyses; + + public DataFrameAnalyticsConfig(String id, String source, String dest, List analyses) { + this.id = ExceptionsHelper.requireNonNull(id, ID); + this.source = ExceptionsHelper.requireNonNull(source, SOURCE); + this.dest = ExceptionsHelper.requireNonNull(dest, DEST); + this.analyses = ExceptionsHelper.requireNonNull(analyses, ANALYSES); + if (analyses.isEmpty()) { + throw new ElasticsearchParseException("One or more analyses is required"); + } + // TODO Add support for multiple analyses + if (analyses.size() > 1) { + throw new UnsupportedOperationException("Does not yet support multiple analyses"); + } + } + + public DataFrameAnalyticsConfig(StreamInput in) throws IOException { + id = in.readString(); + source = in.readString(); + dest = in.readString(); + analyses = in.readList(DataFrameAnalysisConfig::new); + } + + public String getId() { + return id; + } + + public String getSource() { + return source; + } + + public String getDest() { + return dest; + } + + public List getAnalyses() { + return analyses; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + builder.field(SOURCE.getPreferredName(), source); + builder.field(DEST.getPreferredName(), dest); + builder.field(ANALYSES.getPreferredName(), analyses); + if (params.paramAsBoolean(ToXContentParams.INCLUDE_TYPE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(source); + out.writeString(dest); + out.writeList(analyses); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o == null || getClass() != o.getClass()) return false; + + DataFrameAnalyticsConfig other = (DataFrameAnalyticsConfig) o; + return Objects.equals(id, other.id) + && Objects.equals(source, other.source) + && Objects.equals(dest, other.dest) + && Objects.equals(analyses, other.analyses); + } + + @Override + public int hashCode() { + return Objects.hash(id, source, dest, analyses); + } + + public static String documentId(String id) { + return TYPE + "-" + id; + } + + public static class Builder { + + private String id; + private String source; + private String dest; + private List analyses; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = ExceptionsHelper.requireNonNull(id, ID); + } + + public void setSource(String source) { + this.source = ExceptionsHelper.requireNonNull(source, SOURCE); + } + + public void setDest(String dest) { + this.dest = ExceptionsHelper.requireNonNull(dest, DEST); + } + + public void setAnalyses(List analyses) { + this.analyses = ExceptionsHelper.requireNonNull(analyses, ANALYSES); + } + + public DataFrameAnalyticsConfig build() { + return new DataFrameAnalyticsConfig(id, source, dest, analyses); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java new file mode 100644 index 0000000000000..d40df259eec57 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum DataFrameAnalyticsState implements Writeable { + + STARTED, REINDEXING, ANALYZING, STOPPING, STOPPED; + + public static DataFrameAnalyticsState fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static DataFrameAnalyticsState fromStream(StreamInput in) throws IOException { + return in.readEnum(DataFrameAnalyticsState.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(this); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java new file mode 100644 index 0000000000000..27347336cb07c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.MlTasks; + +import java.io.IOException; +import java.util.Objects; + +public class DataFrameAnalyticsTaskState implements PersistentTaskState { + + private static ParseField STATE = new ParseField("state"); + private static ParseField ALLOCATION_ID = new ParseField("allocation_id"); + + private final DataFrameAnalyticsState state; + private final long allocationId; + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, + a -> new DataFrameAnalyticsTaskState((DataFrameAnalyticsState) a[0], (long) a[1])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return DataFrameAnalyticsState.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, STATE, ObjectParser.ValueType.STRING); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), ALLOCATION_ID); + } + + public static DataFrameAnalyticsTaskState fromXContent(XContentParser parser) { + try { + return PARSER.parse(parser, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public DataFrameAnalyticsTaskState(DataFrameAnalyticsState state, long allocationId) { + this.state = Objects.requireNonNull(state); + this.allocationId = allocationId; + } + + public DataFrameAnalyticsState getState() { + return state; + } + + public boolean isStatusStale(PersistentTasksCustomMetaData.PersistentTask task) { + return allocationId != task.getAllocationId(); + } + + @Override + public String getWriteableName() { + return MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + state.writeTo(out); + out.writeLong(allocationId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(STATE.getPreferredName(), state.toString()); + builder.field(ALLOCATION_ID.getPreferredName(), allocationId); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataFrameAnalyticsTaskState that = (DataFrameAnalyticsTaskState) o; + return allocationId == that.allocationId && + state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(state, allocationId); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index d51a8f10e4a5a..6f60a549ae707 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -144,6 +145,7 @@ public static XContentBuilder configMapping() throws IOException { addJobConfigFields(builder); addDatafeedConfigFields(builder); + addDataFrameAnalyticsFields(builder); builder.endObject() .endObject() @@ -386,6 +388,32 @@ public static void addDatafeedConfigFields(XContentBuilder builder) throws IOExc .endObject(); } + public static void addDataFrameAnalyticsFields(XContentBuilder builder) throws IOException { + builder.startObject(DataFrameAnalyticsConfig.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataFrameAnalyticsConfig.SOURCE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataFrameAnalyticsConfig.DEST.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataFrameAnalyticsConfig.ANALYSES.getPreferredName()) + .startObject(PROPERTIES) + .startObject("outlier_detection") + .startObject(PROPERTIES) + .startObject("number_neighbours") + .field(TYPE, INTEGER) + .endObject() + .startObject("method") + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + } + /** * Creates a default mapping which has a dynamic template that * treats all dynamically added fields as keywords. This is needed diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java index b66fd948a5a83..27468a2757858 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java @@ -10,7 +10,7 @@ /** * Interface for classes that write arrays of strings to the - * Ml analytics processes. + * Ml dataframe processes. */ public interface RecordWriter { /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java index 47c0d4f64f96f..c847b679a7493 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java @@ -10,6 +10,7 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.ParseField; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.xpack.core.ml.job.messages.Messages; @@ -86,4 +87,8 @@ public static T requireNonNull(T obj, String paramName) { } return obj; } + + public static T requireNonNull(T obj, ParseField paramName) { + return requireNonNull(obj, paramName.getPreferredName()); + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfigTests.java new file mode 100644 index 0000000000000..a5dc889eea3db --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfigTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class DataFrameAnalysisConfigTests extends AbstractSerializingTestCase { + + @Override + protected DataFrameAnalysisConfig createTestInstance() { + return randomConfig(); + } + + @Override + protected DataFrameAnalysisConfig doParseInstance(XContentParser parser) throws IOException { + return DataFrameAnalysisConfig.parser().parse(parser, null); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataFrameAnalysisConfig::new; + } + + public static DataFrameAnalysisConfig randomConfig() { + Map configParams = new HashMap<>(); + int count = randomIntBetween(1, 5); + for (int i = 0; i < count; i++) { + if (randomBoolean()) { + configParams.put(randomAlphaOfLength(10), randomInt()); + } else { + configParams.put(randomAlphaOfLength(10), randomAlphaOfLength(10)); + } + } + Map config = new HashMap<>(); + config.put(randomAlphaOfLength(10), configParams); + return new DataFrameAnalysisConfig(config); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java new file mode 100644 index 0000000000000..9e983666f0334 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.dataframe; + +import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class DataFrameAnalyticsConfigTests extends AbstractSerializingTestCase { + + @Override + protected DataFrameAnalyticsConfig doParseInstance(XContentParser parser) throws IOException { + return DataFrameAnalyticsConfig.STRICT_PARSER.apply(parser, null).build(); + } + + @Override + protected DataFrameAnalyticsConfig createTestInstance() { + String id = randomValidId(); + String source = randomAlphaOfLength(10); + String dest = randomAlphaOfLength(10); + List analyses = Collections.singletonList(DataFrameAnalysisConfigTests.randomConfig()); + return new DataFrameAnalyticsConfig(id, source, dest, analyses); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataFrameAnalyticsConfig::new; + } + + public static String randomValidId() { + CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray()); + return generator.ofCodePointsLength(random(), 10, 10); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 272415421426f..833436d7ed7ee 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -75,6 +75,8 @@ import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction; import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction; import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; @@ -93,12 +95,13 @@ import org.elasticsearch.xpack.core.ml.action.PostDataAction; import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction; import org.elasticsearch.xpack.core.ml.action.PutCalendarAction; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; import org.elasticsearch.xpack.core.ml.action.PutFilterAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.RunAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; +import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction; @@ -132,6 +135,8 @@ import org.elasticsearch.xpack.ml.action.TransportGetCalendarEventsAction; import org.elasticsearch.xpack.ml.action.TransportGetCalendarsAction; import org.elasticsearch.xpack.ml.action.TransportGetCategoriesAction; +import org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.ml.action.TransportGetDataFrameAnalyticsStatsAction; import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsAction; import org.elasticsearch.xpack.ml.action.TransportGetDatafeedsStatsAction; import org.elasticsearch.xpack.ml.action.TransportGetFiltersAction; @@ -150,12 +155,13 @@ import org.elasticsearch.xpack.ml.action.TransportPostDataAction; import org.elasticsearch.xpack.ml.action.TransportPreviewDatafeedAction; import org.elasticsearch.xpack.ml.action.TransportPutCalendarAction; +import org.elasticsearch.xpack.ml.action.TransportPutDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportPutDatafeedAction; import org.elasticsearch.xpack.ml.action.TransportPutFilterAction; import org.elasticsearch.xpack.ml.action.TransportPutJobAction; import org.elasticsearch.xpack.ml.action.TransportRevertModelSnapshotAction; -import org.elasticsearch.xpack.ml.action.TransportRunAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportSetUpgradeModeAction; +import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportStartDatafeedAction; import org.elasticsearch.xpack.ml.action.TransportStopDatafeedAction; import org.elasticsearch.xpack.ml.action.TransportUpdateCalendarJobAction; @@ -166,12 +172,14 @@ import org.elasticsearch.xpack.ml.action.TransportUpdateProcessAction; import org.elasticsearch.xpack.ml.action.TransportValidateDetectorAction; import org.elasticsearch.xpack.ml.action.TransportValidateJobConfigAction; -import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessFactory; -import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager; -import org.elasticsearch.xpack.ml.dataframe.process.NativeAnalyticsProcessFactory; import org.elasticsearch.xpack.ml.datafeed.DatafeedJobBuilder; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider; +import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager; +import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider; +import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessFactory; +import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager; +import org.elasticsearch.xpack.ml.dataframe.process.NativeAnalyticsProcessFactory; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.JobManagerHolder; import org.elasticsearch.xpack.ml.job.UpdateJobProcessNotifier; @@ -197,7 +205,6 @@ import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.rest.RestFindFileStructureAction; import org.elasticsearch.xpack.ml.rest.RestMlInfoAction; -import org.elasticsearch.xpack.ml.rest.analytics.RestRunAnalyticsAction; import org.elasticsearch.xpack.ml.rest.RestSetUpgradeModeAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarEventAction; @@ -215,6 +222,10 @@ import org.elasticsearch.xpack.ml.rest.datafeeds.RestStartDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestStopDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestUpdateDatafeedAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsStatsAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestPutDataFrameAnalyticsAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestStartDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.filter.RestDeleteFilterAction; import org.elasticsearch.xpack.ml.rest.filter.RestGetFiltersAction; import org.elasticsearch.xpack.ml.rest.filter.RestPutFilterAction; @@ -291,6 +302,7 @@ public class MachineLearning extends Plugin implements ActionPlugin, AnalysisPlu private final SetOnce autodetectProcessManager = new SetOnce<>(); private final SetOnce datafeedManager = new SetOnce<>(); + private final SetOnce dataFrameAnalyticsManager = new SetOnce<>(); private final SetOnce memoryTracker = new SetOnce<>(); public MachineLearning(Settings settings, Path configPath) { @@ -455,8 +467,13 @@ public Collection createComponents(Client client, ClusterService cluster // run node startup tasks autodetectProcessManager.onNodeStartup(); + // Data frame analytics components AnalyticsProcessManager analyticsProcessManager = new AnalyticsProcessManager(client, environment, threadPool, analyticsProcessFactory); + DataFrameAnalyticsConfigProvider dataFrameAnalyticsConfigProvider = new DataFrameAnalyticsConfigProvider(client); + DataFrameAnalyticsManager dataFrameAnalyticsManager = new DataFrameAnalyticsManager(clusterService, client, + dataFrameAnalyticsConfigProvider, analyticsProcessManager); + this.dataFrameAnalyticsManager.set(dataFrameAnalyticsManager); return Arrays.asList( mlLifeCycleService, @@ -472,7 +489,8 @@ public Collection createComponents(Client client, ClusterService cluster auditor, new MlAssignmentNotifier(settings, auditor, threadPool, client, clusterService), memoryTracker, - analyticsProcessManager + analyticsProcessManager, + dataFrameAnalyticsConfigProvider ); } @@ -487,7 +505,8 @@ public List> getPersistentTasksExecutor(ClusterServic return Arrays.asList( new TransportOpenJobAction.OpenJobPersistentTasksExecutor(settings, clusterService, autodetectProcessManager.get(), memoryTracker.get(), client), - new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedManager.get()) + new TransportStartDatafeedAction.StartDatafeedPersistentTasksExecutor(datafeedManager.get()), + new TransportStartDataFrameAnalyticsAction.TaskExecutor(dataFrameAnalyticsManager.get()) ); } @@ -559,8 +578,11 @@ public List getRestHandlers(Settings settings, RestController restC new RestGetCalendarEventsAction(settings, restController), new RestPostCalendarEventAction(settings, restController), new RestFindFileStructureAction(settings, restController), - new RestRunAnalyticsAction(settings, restController), - new RestSetUpgradeModeAction(settings, restController) + new RestSetUpgradeModeAction(settings, restController), + new RestGetDataFrameAnalyticsAction(settings, restController), + new RestGetDataFrameAnalyticsStatsAction(settings, restController), + new RestPutDataFrameAnalyticsAction(settings, restController), + new RestStartDataFrameAnalyticsAction(settings, restController) ); } @@ -619,8 +641,11 @@ public List getRestHandlers(Settings settings, RestController restC new ActionHandler<>(PostCalendarEventsAction.INSTANCE, TransportPostCalendarEventsAction.class), new ActionHandler<>(PersistJobAction.INSTANCE, TransportPersistJobAction.class), new ActionHandler<>(FindFileStructureAction.INSTANCE, TransportFindFileStructureAction.class), - new ActionHandler<>(RunAnalyticsAction.INSTANCE, TransportRunAnalyticsAction.class), - new ActionHandler<>(SetUpgradeModeAction.INSTANCE, TransportSetUpgradeModeAction.class) + new ActionHandler<>(SetUpgradeModeAction.INSTANCE, TransportSetUpgradeModeAction.class), + new ActionHandler<>(GetDataFrameAnalyticsAction.INSTANCE, TransportGetDataFrameAnalyticsAction.class), + new ActionHandler<>(GetDataFrameAnalyticsStatsAction.INSTANCE, TransportGetDataFrameAnalyticsStatsAction.class), + new ActionHandler<>(PutDataFrameAnalyticsAction.INSTANCE, TransportPutDataFrameAnalyticsAction.class), + new ActionHandler<>(StartDataFrameAnalyticsAction.INSTANCE, TransportStartDataFrameAnalyticsAction.class) ); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java new file mode 100644 index 0000000000000..616d22116312f --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.action.AbstractGetResourcesRequest; +import org.elasticsearch.xpack.core.ml.action.AbstractGetResourcesResponse; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.utils.MlIndicesUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public abstract class AbstractTransportGetResourcesAction> + extends HandledTransportAction { + + private Client client; + + protected AbstractTransportGetResourcesAction(String actionName, TransportService transportService, ActionFilters actionFilters, + Supplier request, Client client) { + super(actionName, transportService, actionFilters, request); + this.client = Objects.requireNonNull(client); + } + + protected void searchResources(AbstractGetResourcesRequest request, ActionListener> listener) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .sort(request.getResourceIdField()) + .from(request.getPageParams().getFrom()) + .size(request.getPageParams().getSize()) + .query(buildQuery(request)); + + SearchRequest searchRequest = new SearchRequest(getIndices()) + .indicesOptions(MlIndicesUtils.addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS)) + .source(sourceBuilder); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, new ActionListener() { + @Override + public void onResponse(SearchResponse response) { + List docs = new ArrayList<>(); + for (SearchHit hit : response.getHits().getHits()) { + BytesReference docSource = hit.getSourceRef(); + try (InputStream stream = docSource.streamInput(); + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { + docs.add(parse(parser)); + } catch (IOException e) { + this.onFailure(e); + } + } + + if (docs.isEmpty() && Regex.isSimpleMatchPattern(request.getResourceId()) == false) { + listener.onFailure(notFoundException(request.getResourceId())); + } else { + listener.onResponse(new QueryPage<>(docs, docs.size(), getResultsField())); + } + } + + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, + client::search); + } + + private QueryBuilder buildQuery(AbstractGetResourcesRequest request) { + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + if (Strings.isNullOrEmpty(request.getResourceId()) == false) { + boolQuery.filter(QueryBuilders.wildcardQuery(request.getResourceIdField(), request.getResourceId())); + } + QueryBuilder additionalQuery = additionalQuery(); + if (additionalQuery != null) { + boolQuery.filter(additionalQuery); + } + return boolQuery.hasClauses() ? boolQuery : QueryBuilders.matchAllQuery(); + } + + @Nullable + protected QueryBuilder additionalQuery() { + return null; + } + + protected abstract ParseField getResultsField(); + + protected abstract String[] getIndices(); + + protected abstract Resource parse(XContentParser parser); + + protected abstract ResourceNotFoundException notFoundException(String resourceId); +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..7b0697895c3c0 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; + +public class TransportGetDataFrameAnalyticsAction extends AbstractTransportGetResourcesAction { + + @Inject + public TransportGetDataFrameAnalyticsAction(TransportService transportService, ActionFilters actionFilters, Client client) { + super(GetDataFrameAnalyticsAction.NAME, transportService, actionFilters, GetDataFrameAnalyticsAction.Request::new, client); + } + + @Override + protected ParseField getResultsField() { + return GetDataFrameAnalyticsAction.Response.CONFIGS; + } + + @Override + protected String[] getIndices() { + return new String[] { AnomalyDetectorsIndex.configIndexName() }; + } + + @Override + protected DataFrameAnalyticsConfig parse(XContentParser parser) { + return DataFrameAnalyticsConfig.LENIENT_PARSER.apply(parser, null).build(); + } + + @Override + protected ResourceNotFoundException notFoundException(String resourceId) { + return new ResourceNotFoundException("No known data frame analytics with id [{}]", resourceId); + } + + @Override + protected void doExecute(Task task, GetDataFrameAnalyticsAction.Request request, + ActionListener listener) { + searchResources(request, ActionListener.wrap( + queryPage -> listener.onResponse(new GetDataFrameAnalyticsAction.Response(queryPage)), + listener::onFailure + )); + } + + @Nullable + protected QueryBuilder additionalQuery() { + return QueryBuilders.termQuery(DataFrameAnalyticsConfig.CONFIG_TYPE.getPreferredName(), DataFrameAnalyticsConfig.TYPE); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java new file mode 100644 index 0000000000000..218e07bd3757f --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public class TransportGetDataFrameAnalyticsStatsAction + extends TransportMasterNodeAction { + + private final Client client; + + @Inject + public TransportGetDataFrameAnalyticsStatsAction(TransportService transportService, ClusterService clusterService, Client client, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(GetDataFrameAnalyticsStatsAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, GetDataFrameAnalyticsStatsAction.Request::new); + this.client = client; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected GetDataFrameAnalyticsStatsAction.Response newResponse() { + return new GetDataFrameAnalyticsStatsAction.Response(); + } + + @Override + protected void masterOperation(GetDataFrameAnalyticsStatsAction.Request request, ClusterState state, + ActionListener listener) throws Exception { + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + + ActionListener getResponseListener = ActionListener.wrap( + response -> { + List stats = new ArrayList(response.getResources().results().size()); + response.getResources().results().forEach(c -> stats.add(buildStats(c.getId(), tasks, state))); + listener.onResponse(new GetDataFrameAnalyticsStatsAction.Response(new QueryPage<>(stats, stats.size(), + new ParseField("stats")))); + }, + listener::onFailure + ); + + GetDataFrameAnalyticsAction.Request getRequest = new GetDataFrameAnalyticsAction.Request(); + getRequest.setResourceId(request.getId()); + getRequest.setPageParams(request.getPageParams()); + executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, getRequest, getResponseListener); + + } + + private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId, PersistentTasksCustomMetaData tasks, + ClusterState clusterState) { + PersistentTasksCustomMetaData.PersistentTask analyticsTask = MlTasks.getDataFrameAnalyticsTask(concreteAnalyticsId, tasks); + DataFrameAnalyticsState analyticsState = MlTasks.getDataFrameAnalyticsState(concreteAnalyticsId, tasks); + DiscoveryNode node = null; + String assignmentExplanation = null; + if (analyticsTask != null) { + node = clusterState.nodes().get(analyticsTask.getExecutorNode()); + assignmentExplanation = analyticsTask.getAssignment().getExplanation(); + } + return new GetDataFrameAnalyticsStatsAction.Response.Stats( + concreteAnalyticsId, analyticsState, node, assignmentExplanation); + } + + @Override + protected ClusterBlockException checkBlock(GetDataFrameAnalyticsStatsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..33e8d8e6c6bfc --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.MlStrings; +import org.elasticsearch.xpack.ml.dataframe.analyses.DataFrameAnalysesUtils; +import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider; + +import java.util.function.Supplier; + +public class TransportPutDataFrameAnalyticsAction + extends HandledTransportAction { + + private final XPackLicenseState licenseState; + private final DataFrameAnalyticsConfigProvider configProvider; + + @Inject + public TransportPutDataFrameAnalyticsAction(TransportService transportService, ActionFilters actionFilters, + XPackLicenseState licenseState, DataFrameAnalyticsConfigProvider configProvider) { + super(PutDataFrameAnalyticsAction.NAME, transportService, actionFilters, + (Supplier) PutDataFrameAnalyticsAction.Request::new); + this.licenseState = licenseState; + this.configProvider = configProvider; + } + + @Override + protected void doExecute(Task task, PutDataFrameAnalyticsAction.Request request, + ActionListener listener) { + if (licenseState.isMachineLearningAllowed() == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); + return; + } + + validateConfig(request.getConfig()); + configProvider.put(request.getConfig(), ActionListener.wrap( + indexResponse -> listener.onResponse(new PutDataFrameAnalyticsAction.Response(true)), + listener::onFailure + )); + } + + private void validateConfig(DataFrameAnalyticsConfig config) { + if (MlStrings.isValidId(config.getId()) == false) { + throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.INVALID_ID, DataFrameAnalyticsConfig.ID, + config.getId())); + } + if (!MlStrings.hasValidLengthForId(config.getId())) { + throw ExceptionsHelper.badRequestException("id [{}] is too long; must not contain more than {} characters", config.getId(), + MlStrings.ID_LENGTH_LIMIT); + } + if (config.getSource().isEmpty()) { + throw ExceptionsHelper.badRequestException("[{}] must be non-empty", DataFrameAnalyticsConfig.SOURCE); + } + if (config.getDest().isEmpty()) { + throw ExceptionsHelper.badRequestException("[{}] must be non-empty", DataFrameAnalyticsConfig.DEST); + } + DataFrameAnalysesUtils.readAnalyses(config.getAnalyses()); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..f100e8f91f26b --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsManager; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory; +import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; + +public class TransportStartDataFrameAnalyticsAction + extends TransportMasterNodeAction { + + private final XPackLicenseState licenseState; + private final Client client; + private final PersistentTasksService persistentTasksService; + private final DataFrameAnalyticsConfigProvider configProvider; + + @Inject + public TransportStartDataFrameAnalyticsAction(TransportService transportService, Client client, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, XPackLicenseState licenseState, + IndexNameExpressionResolver indexNameExpressionResolver, + PersistentTasksService persistentTasksService, + DataFrameAnalyticsConfigProvider configProvider) { + super(StartDataFrameAnalyticsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, + StartDataFrameAnalyticsAction.Request::new); + this.licenseState = licenseState; + this.client = client; + this.persistentTasksService = persistentTasksService; + this.configProvider = configProvider; + } + + @Override + protected String executor() { + // This api doesn't do heavy or blocking operations (just delegates PersistentTasksService), + // so we can do this on the network thread + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse newResponse() { + return new AcknowledgedResponse(); + } + + @Override + protected ClusterBlockException checkBlock(StartDataFrameAnalyticsAction.Request request, ClusterState state) { + // We only delegate here to PersistentTasksService, but if there is a metadata writeblock, + // then delegating to PersistentTasksService doesn't make a whole lot of sense, + // because PersistentTasksService will then fail. + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected void masterOperation(StartDataFrameAnalyticsAction.Request request, ClusterState state, + ActionListener listener) { + if (licenseState.isMachineLearningAllowed() == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); + return; + } + + StartDataFrameAnalyticsAction.TaskParams taskParams = new StartDataFrameAnalyticsAction.TaskParams(request.getId()); + + // Wait for analytics to be started + ActionListener> waitForAnalyticsToStart = + new ActionListener>() { + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask task) { + listener.onResponse(new AcknowledgedResponse(true)); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + e = new ElasticsearchStatusException("Cannot open data frame analytics [" + request.getId() + + "] because it has already been opened", RestStatus.CONFLICT, e); + } + listener.onFailure(e); + } + }; + + // Start persistent task + ActionListener validatedConfigListener = ActionListener.wrap( + config -> persistentTasksService.sendStartRequest(MlTasks.dataFrameAnalyticsTaskId(request.getId()), + MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, taskParams, waitForAnalyticsToStart), + listener::onFailure + ); + + // Validate config + ActionListener configListener = ActionListener.wrap( + config -> DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getSource(), ActionListener.wrap( + dataExtractorFactory -> validatedConfigListener.onResponse(config), listener::onFailure)), + listener::onFailure + ); + + // Get config + configProvider.get(request.getId(), configListener); + } + + public static class DataFrameAnalyticsTask extends AllocatedPersistentTask { + + private final StartDataFrameAnalyticsAction.TaskParams taskParams; + + public DataFrameAnalyticsTask(long id, String type, String action, TaskId parentTask, Map headers, + StartDataFrameAnalyticsAction.TaskParams taskParams) { + super(id, type, action, "data_frame_analytics-" + taskParams.getId(), parentTask, headers); + this.taskParams = Objects.requireNonNull(taskParams); + } + + public StartDataFrameAnalyticsAction.TaskParams getParams() { + return taskParams; + } + } + + public static class TaskExecutor extends PersistentTasksExecutor { + + private final DataFrameAnalyticsManager manager; + + public TaskExecutor(DataFrameAnalyticsManager manager) { + super(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, MachineLearning.UTILITY_THREAD_POOL_NAME); + this.manager = Objects.requireNonNull(manager); + } + + @Override + protected AllocatedPersistentTask createTask( + long id, String type, String action, TaskId parentTaskId, + PersistentTasksCustomMetaData.PersistentTask persistentTask, + Map headers) { + return new DataFrameAnalyticsTask(id, type, action, parentTaskId, headers, persistentTask.getParams()); + } + + @Override + protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { + // For starters, let's just select the least loaded ML node + // TODO implement memory-based load balancing + return super.selectLeastLoadedNode(clusterState, MachineLearning::isMlNode); + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, StartDataFrameAnalyticsAction.TaskParams params, + PersistentTaskState state) { + DataFrameAnalyticsTaskState startedState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.STARTED, + task.getAllocationId()); + task.updatePersistentTaskState(startedState, ActionListener.wrap( + response -> manager.execute((DataFrameAnalyticsTask) task), + task::markAsFailed + )); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalysis.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalysis.java deleted file mode 100644 index 81062f9795040..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalysis.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.dataframe; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; - -import java.io.IOException; - -public class DataFrameAnalysis implements ToXContentObject { - - private static final ParseField NAME = new ParseField("name"); - - private final String name; - - public DataFrameAnalysis(String name) { - this.name = ExceptionsHelper.requireNonNull(name, NAME.getPreferredName()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(NAME.getPreferredName(), name); - builder.endObject(); - return builder; - } -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameFields.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java similarity index 79% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameFields.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java index 164b7888a6ffe..eeb3a8badce39 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameFields.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsFields.java @@ -5,9 +5,9 @@ */ package org.elasticsearch.xpack.ml.dataframe; -public final class DataFrameFields { +public final class DataFrameAnalyticsFields { public static final String ID = "_id_copy"; - private DataFrameFields() {} + private DataFrameAnalyticsFields() {} } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRunAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java similarity index 50% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRunAnalyticsAction.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index cb9e25504a8e4..18f0bb647c245 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRunAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -3,28 +3,21 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.action; +package org.elasticsearch.xpack.ml.dataframe; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.reindex.BulkByScrollResponse; @@ -32,14 +25,12 @@ import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.script.Script; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.ml.action.RunAnalyticsAction; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.dataframe.DataFrameDataExtractorFactory; -import org.elasticsearch.xpack.ml.dataframe.DataFrameFields; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsTaskState; +import org.elasticsearch.xpack.ml.action.TransportStartDataFrameAnalyticsAction.DataFrameAnalyticsTask; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory; +import org.elasticsearch.xpack.ml.dataframe.persistence.DataFrameAnalyticsConfigProvider; import org.elasticsearch.xpack.ml.dataframe.process.AnalyticsProcessManager; import java.util.Arrays; @@ -47,16 +38,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Supplier; +import java.util.Objects; -public class TransportRunAnalyticsAction extends HandledTransportAction { - - private final TransportService transportService; - private final ThreadPool threadPool; - private final Client client; - private final ClusterService clusterService; - private final Environment environment; - private final AnalyticsProcessManager analyticsProcessManager; +public class DataFrameAnalyticsManager { /** * Unfortunately, getting the settings of an index include internal settings that should @@ -70,67 +54,95 @@ public class TransportRunAnalyticsAction extends HandledTransportAction) RunAnalyticsAction.Request::new); - this.transportService = transportService; - this.threadPool = threadPool; - this.client = client; - this.clusterService = clusterService; - this.environment = environment; - this.analyticsProcessManager = analyticsProcessManager; + private final ClusterService clusterService; + private final Client client; + private final DataFrameAnalyticsConfigProvider configProvider; + private final AnalyticsProcessManager processManager; + + public DataFrameAnalyticsManager(ClusterService clusterService, Client client, DataFrameAnalyticsConfigProvider configProvider, + AnalyticsProcessManager processManager) { + this.clusterService = Objects.requireNonNull(clusterService); + this.client = Objects.requireNonNull(client); + this.configProvider = Objects.requireNonNull(configProvider); + this.processManager = Objects.requireNonNull(processManager); } - @Override - protected void doExecute(Task task, RunAnalyticsAction.Request request, ActionListener listener) { - DiscoveryNode localNode = clusterService.localNode(); - if (MachineLearning.isMlNode(localNode)) { - reindexDataframeAndStartAnalysis(request.getIndex(), listener); - return; - } + public void execute(DataFrameAnalyticsTask task) { + ActionListener reindexingStateListener = ActionListener.wrap( + config -> reindexDataframeAndStartAnalysis(task, config), + e -> task.markAsFailed(e) + ); - ClusterState clusterState = clusterService.state(); - for (DiscoveryNode node : clusterState.getNodes()) { - if (MachineLearning.isMlNode(node)) { - transportService.sendRequest(node, actionName, request, - new ActionListenerResponseHandler<>(listener, inputStream -> { - AcknowledgedResponse response = new AcknowledgedResponse(); - response.readFrom(inputStream); - return response; - })); - return; - } - } - listener.onFailure(ExceptionsHelper.badRequestException("No ML node to run on")); + // Update task state to REINDEXING + ActionListener configListener = ActionListener.wrap( + config -> { + DataFrameAnalyticsTaskState reindexingState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.REINDEXING, + task.getAllocationId()); + task.updatePersistentTaskState(reindexingState, ActionListener.wrap( + updatedTask -> reindexingStateListener.onResponse(config), + reindexingStateListener::onFailure + )); + }, + reindexingStateListener::onFailure + ); + + // Retrieve configuration + configProvider.get(task.getParams().getId(), configListener); } - private void reindexDataframeAndStartAnalysis(String index, ActionListener listener) { - final String destinationIndex = index + "_copy"; + private void reindexDataframeAndStartAnalysis(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config) { + // Reindexing is complete; start analytics + ActionListener refreshListener = ActionListener.wrap( + refreshResponse -> startAnalytics(task, config), + task::markAsFailed + ); + // Refresh to ensure copied index is fully searchable ActionListener reindexCompletedListener = ActionListener.wrap( - bulkResponse -> { - client.execute(RefreshAction.INSTANCE, new RefreshRequest(destinationIndex), ActionListener.wrap( - refreshResponse -> { - runPipelineAnalytics(destinationIndex, listener); - }, listener::onFailure - )); - }, listener::onFailure + bulkResponse -> client.execute(RefreshAction.INSTANCE, new RefreshRequest(config.getDest()), refreshListener), + e -> task.markAsFailed(e) ); + // Reindex ActionListener copyIndexCreatedListener = ActionListener.wrap( createIndexResponse -> { ReindexRequest reindexRequest = new ReindexRequest(); - reindexRequest.setSourceIndices(index); - reindexRequest.setDestIndex(destinationIndex); - reindexRequest.setScript(new Script("ctx._source." + DataFrameFields.ID + " = ctx._id")); + reindexRequest.setSourceIndices(config.getSource()); + reindexRequest.setDestIndex(config.getDest()); + reindexRequest.setScript(new Script("ctx._source." + DataFrameAnalyticsFields.ID + " = ctx._id")); client.execute(ReindexAction.INSTANCE, reindexRequest, reindexCompletedListener); - }, listener::onFailure + }, + reindexCompletedListener::onFailure ); - createDestinationIndex(index, destinationIndex, copyIndexCreatedListener); + createDestinationIndex(config.getSource(), config.getDest(), copyIndexCreatedListener); + } + + private void startAnalytics(DataFrameAnalyticsTask task, DataFrameAnalyticsConfig config) { + // Update state to ANALYZING and start process + ActionListener dataExtractorFactoryListener = ActionListener.wrap( + dataExtractorFactory -> { + DataFrameAnalyticsTaskState analyzingState = new DataFrameAnalyticsTaskState(DataFrameAnalyticsState.ANALYZING, + task.getAllocationId()); + task.updatePersistentTaskState(analyzingState, ActionListener.wrap( + updatedTask -> processManager.runJob(config, dataExtractorFactory, + error -> { + if (error != null) { + task.markAsFailed(error); + } else { + task.markAsCompleted(); + } + }), + task::markAsFailed + )); + }, + e -> task.markAsFailed(e) + ); + + // TODO This could fail with errors. In that case we get stuck with the copied index. + // We could delete the index in case of failure or we could try building the factory before reindexing + // to catch the error early on. + DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getDest(), dataExtractorFactoryListener); } private void createDestinationIndex(String sourceIndex, String destinationIndex, ActionListener listener) { @@ -140,14 +152,9 @@ private void createDestinationIndex(String sourceIndex, String destinationIndex, return; } - if (indexMetaData.getMappings().size() != 1) { - listener.onFailure(ExceptionsHelper.badRequestException("Does not support indices with multiple types")); - return; - } - Settings.Builder settingsBuilder = Settings.builder().put(indexMetaData.getSettings()); INTERNAL_SETTINGS.stream().forEach(settingsBuilder::remove); - settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataFrameFields.ID); + settingsBuilder.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataFrameAnalyticsFields.ID); settingsBuilder.put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), SortOrder.ASC); CreateIndexRequest createIndexRequest = new CreateIndexRequest(destinationIndex, settingsBuilder.build()); @@ -161,25 +168,8 @@ private static void addDestinationIndexMappings(IndexMetaData indexMetaData, Cre Map properties = (Map) mappingsAsMap.get("properties"); Map idCopyMapping = new HashMap<>(); idCopyMapping.put("type", "keyword"); - properties.put(DataFrameFields.ID, idCopyMapping); + properties.put(DataFrameAnalyticsFields.ID, idCopyMapping); createIndexRequest.mapping(mappings.keysIt().next(), mappingsAsMap); } - - private void runPipelineAnalytics(String index, ActionListener listener) { - String jobId = "ml-analytics-" + index; - - ActionListener dataExtractorFactoryListener = ActionListener.wrap( - dataExtractorFactory -> { - analyticsProcessManager.runJob(jobId, dataExtractorFactory); - listener.onResponse(new AcknowledgedResponse(true)); - }, - listener::onFailure - ); - - // TODO This could fail with errors. In that case we get stuck with the copied index. - // We could delete the index in case of failure or we could try building the factory before reindexing - // to catch the error early on. - DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), index, dataExtractorFactoryListener); - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java new file mode 100644 index 0000000000000..a941c1609a287 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.analyses; + +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public abstract class AbstractDataFrameAnalysis implements DataFrameAnalysis { + + private static final String NAME = "name"; + private static final String PARAMETERS = "parameters"; + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NAME, typeToConfigName(getType())); + builder.field(PARAMETERS, getParams()); + builder.endObject(); + return builder; + } + + // TODO Make the c++ analyses names same as in java to get rid of this + private String typeToConfigName(Type type) { + switch (type) { + case OUTLIER_DETECTION: + return "outliers"; + default: + throw new IllegalStateException("Unexpected type: " + type); + } + } + + protected abstract Map getParams(); +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java new file mode 100644 index 0000000000000..3a02932c39c41 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.analyses; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalysisConfig; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public final class DataFrameAnalysesUtils { + + private static final Map factories; + + static { + factories = new HashMap<>(); + factories.put(DataFrameAnalysis.Type.OUTLIER_DETECTION, new OutlierDetection.Factory()); + } + + private DataFrameAnalysesUtils() {} + + public static List readAnalyses(List analyses) { + return analyses.stream().map(DataFrameAnalysesUtils::readAnalysis).collect(Collectors.toList()); + } + + static DataFrameAnalysis readAnalysis(DataFrameAnalysisConfig config) { + Map configMap = config.asMap(); + DataFrameAnalysis.Type analysisType = DataFrameAnalysis.Type.fromString(configMap.keySet().iterator().next()); + DataFrameAnalysis.Factory factory = factories.get(analysisType); + if (factory == null) { + throw new ElasticsearchParseException("Unknown analysis type [{}]", analysisType); + } + Map analysisConfig = castAsMapAndCopy(analysisType, configMap.get(analysisType.toString())); + DataFrameAnalysis dataFrameAnalysis = factory.create(analysisConfig); + if (analysisConfig.isEmpty() == false) { + throw new ElasticsearchParseException("Data frame analysis [{}] does not support one or more provided parameters: {}", + analysisType, analysisConfig.keySet()); + } + return dataFrameAnalysis; + } + + private static Map castAsMapAndCopy(DataFrameAnalysis.Type analysisType, Object obj) { + try { + return new HashMap<>((Map) obj); + } catch (ClassCastException e) { + throw new ElasticsearchParseException("[{}] expected to be a map but was of type [{}]", analysisType, obj.getClass().getName()); + } + } + + @Nullable + static Integer readInt(DataFrameAnalysis.Type analysisType, Map config, String property) { + Object value = config.remove(property); + if (value == null) { + return null; + } + try { + return (int) value; + } catch (ClassCastException e) { + throw new ElasticsearchParseException("Property [{}] of analysis [{}] should be of type int but was [{}]", + property, analysisType, value.getClass().getName()); + } + } + + @Nullable + static String readString(DataFrameAnalysis.Type analysisType, Map config, String property) { + Object value = config.remove(property); + if (value == null) { + return null; + } + try { + return (String) value; + } catch (ClassCastException e) { + throw new ElasticsearchParseException("Property [{}] of analysis [{}] should be of string int but was [{}]", + property, analysisType, value.getClass().getName()); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysis.java new file mode 100644 index 0000000000000..f1b2a72df3ff2 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysis.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.analyses; + +import org.elasticsearch.common.xcontent.ToXContentObject; + +import java.util.Locale; +import java.util.Map; + +public interface DataFrameAnalysis extends ToXContentObject { + + enum Type { + OUTLIER_DETECTION; + + public static Type fromString(String value) { + return Type.valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + Type getType(); + + interface Factory { + + /** + * Creates a data frame analysis based on the specified map of maps config. + * + * @param config The configuration for the analysis + * + * Note: Implementations are responsible for removing the used configuration keys, so that after + * creation it can be verified that all configurations settings have been used. + */ + DataFrameAnalysis create(Map config); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetection.java new file mode 100644 index 0000000000000..47f614ba658f6 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetection.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.analyses; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +public class OutlierDetection extends AbstractDataFrameAnalysis { + + public enum Method { + LOF, LDOF, DISTANCE_KTH_NN, DISTANCE_KNN; + + public static Method fromString(String value) { + return Method.valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final String NUMBER_NEIGHBOURS = "number_neighbours"; + public static final String METHOD = "method"; + + private final Integer numberNeighbours; + private final Method method; + + public OutlierDetection(Integer numberNeighbours, Method method) { + this.numberNeighbours = numberNeighbours; + this.method = method; + } + + @Override + public Type getType() { + return Type.OUTLIER_DETECTION; + } + + @Override + protected Map getParams() { + Map params = new HashMap<>(); + if (numberNeighbours != null) { + params.put(NUMBER_NEIGHBOURS, numberNeighbours); + } + if (method != null) { + params.put(METHOD, method); + } + return params; + } + + static class Factory implements DataFrameAnalysis.Factory { + + @Override + public DataFrameAnalysis create(Map config) { + Integer numberNeighbours = DataFrameAnalysesUtils.readInt(Type.OUTLIER_DETECTION, config, NUMBER_NEIGHBOURS); + String method = DataFrameAnalysesUtils.readString(Type.OUTLIER_DETECTION, config, METHOD); + return new OutlierDetection(numberNeighbours, method == null ? null : Method.fromString(method)); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java similarity index 97% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractor.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java index 3d17ff7afd2c5..6071c0b111d02 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.dataframe; +package org.elasticsearch.xpack.ml.dataframe.extractor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils; import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField; +import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsFields; import java.io.IOException; import java.util.ArrayList; @@ -95,7 +96,7 @@ protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequest private SearchRequestBuilder buildSearchRequest() { SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) .setScroll(SCROLL_TIMEOUT) - .addSort(DataFrameFields.ID, SortOrder.ASC) + .addSort(DataFrameAnalyticsFields.ID, SortOrder.ASC) .setIndices(context.indices) .setSize(context.scrollSize) .setQuery(context.query) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java similarity index 95% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorContext.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java index 82de257ccaffd..f602a66221f7c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.dataframe; +package org.elasticsearch.xpack.ml.dataframe.extractor; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedFields; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java similarity index 98% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactory.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java index 3080cd4b43a27..a3d6b683e3514 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.dataframe; +package org.elasticsearch.xpack.ml.dataframe.extractor; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -65,7 +65,7 @@ private DataFrameDataExtractorFactory(Client client, String index, ExtractedFiel public DataFrameDataExtractor newExtractor(boolean includeSource) { DataFrameDataExtractorContext context = new DataFrameDataExtractorContext( - "ml-analytics-" + index, + "ml-dataframe-" + index, extractedFields, Arrays.asList(index), QueryBuilders.matchAllQuery(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java new file mode 100644 index 0000000000000..39c149ee02fbf --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +public class DataFrameAnalyticsConfigProvider { + + private static final Map TO_XCONTENT_PARAMS; + + static { + Map modifiable = new HashMap<>(); + modifiable.put(ToXContentParams.INCLUDE_TYPE, "true"); + TO_XCONTENT_PARAMS = Collections.unmodifiableMap(modifiable); + } + + private final Client client; + + public DataFrameAnalyticsConfigProvider(Client client) { + this.client = Objects.requireNonNull(client); + } + + public void put(DataFrameAnalyticsConfig config, ActionListener listener) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + config.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); + IndexRequest indexRequest = new IndexRequest(AnomalyDetectorsIndex.configIndexName()) + .id(DataFrameAnalyticsConfig.documentId(config.getId())) + .opType(DocWriteRequest.OpType.CREATE) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(builder); + + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + } catch (IOException e) { + listener.onFailure(new ElasticsearchParseException("Failed to serialise data frame analytics with id [" + config.getId() + + "]")); + } + } + + public void get(String id, ActionListener listener) { + GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request(); + request.setResourceId(id); + executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, request, ActionListener.wrap( + response -> { + List analytics = response.getResources().results(); + if (analytics.size() != 1) { + listener.onFailure(ExceptionsHelper.badRequestException("Expected a single match for data frame analytics [{}] " + + "but got [{}]", id, analytics.size())); + } else { + listener.onResponse(analytics.get(0)); + } + }, + listener::onFailure + )); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java index 0d925ed439d3f..a96bb593346b5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java @@ -20,7 +20,7 @@ public interface AnalyticsProcess extends NativeProcess { void writeEndOfDataMessage() throws IOException; /** - * @return stream of analytics results. + * @return stream of dataframe results. */ Iterator readAnalyticsResults(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java index ce186b9232da3..36507e3da292b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessConfig.java @@ -8,7 +8,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalysis; +import org.elasticsearch.xpack.ml.dataframe.analyses.DataFrameAnalysis; import java.io.IOException; import java.util.Objects; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java index 987335d22f0a6..fe98d498d0039 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java @@ -13,17 +13,20 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.MachineLearning; -import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalysis; -import org.elasticsearch.xpack.ml.dataframe.DataFrameDataExtractor; -import org.elasticsearch.xpack.ml.dataframe.DataFrameDataExtractorFactory; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory; +import org.elasticsearch.xpack.ml.dataframe.analyses.DataFrameAnalysesUtils; +import org.elasticsearch.xpack.ml.dataframe.analyses.DataFrameAnalysis; import java.io.IOException; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; public class AnalyticsProcessManager { @@ -42,19 +45,20 @@ public AnalyticsProcessManager(Client client, Environment environment, ThreadPoo this.processFactory = Objects.requireNonNull(analyticsProcessFactory); } - public void runJob(String jobId, DataFrameDataExtractorFactory dataExtractorFactory) { + public void runJob(DataFrameAnalyticsConfig config, DataFrameDataExtractorFactory dataExtractorFactory, + Consumer finishHandler) { threadPool.generic().execute(() -> { DataFrameDataExtractor dataExtractor = dataExtractorFactory.newExtractor(false); - AnalyticsProcess process = createProcess(jobId, createProcessConfig(dataExtractor)); + AnalyticsProcess process = createProcess(config.getId(), createProcessConfig(config, dataExtractor)); ExecutorService executorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME); AnalyticsResultProcessor resultProcessor = new AnalyticsResultProcessor(client, dataExtractorFactory.newExtractor(true)); executorService.execute(() -> resultProcessor.process(process)); - executorService.execute(() -> processData(jobId, dataExtractor, process, resultProcessor)); + executorService.execute(() -> processData(config.getId(), dataExtractor, process, resultProcessor, finishHandler)); }); } private void processData(String jobId, DataFrameDataExtractor dataExtractor, AnalyticsProcess process, - AnalyticsResultProcessor resultProcessor) { + AnalyticsResultProcessor resultProcessor, Consumer finishHandler) { try { writeHeaderRecord(dataExtractor, process); writeDataRows(dataExtractor, process); @@ -66,13 +70,18 @@ private void processData(String jobId, DataFrameDataExtractor dataExtractor, Ana LOGGER.info("[{}] Result processor has completed", jobId); } catch (IOException e) { LOGGER.error(new ParameterizedMessage("[{}] Error writing data to the process", jobId), e); + finishHandler.accept(e); } finally { LOGGER.info("[{}] Closing process", jobId); try { process.close(); LOGGER.info("[{}] Closed process", jobId); + + // This results in marking the persistent task as complete + finishHandler.accept(null); } catch (IOException e) { LOGGER.error("[{}] Error closing data frame analyzer process", jobId); + finishHandler.accept(e); } } } @@ -119,15 +128,19 @@ private AnalyticsProcess createProcess(String jobId, AnalyticsProcessConfig anal ExecutorService executorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME); AnalyticsProcess process = processFactory.createAnalyticsProcess(jobId, analyticsProcessConfig, executorService); if (process.isProcessAlive() == false) { - throw ExceptionsHelper.serverError("Failed to start analytics process"); + throw ExceptionsHelper.serverError("Failed to start dataframe process"); } return process; } - private AnalyticsProcessConfig createProcessConfig(DataFrameDataExtractor dataExtractor) { + private AnalyticsProcessConfig createProcessConfig(DataFrameAnalyticsConfig config, DataFrameDataExtractor dataExtractor) { DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary(); - AnalyticsProcessConfig config = new AnalyticsProcessConfig(dataSummary.rows, dataSummary.cols, - new ByteSizeValue(1, ByteSizeUnit.GB), 1, new DataFrameAnalysis("outliers")); - return config; + List dataFrameAnalyses = DataFrameAnalysesUtils.readAnalyses(config.getAnalyses()); + // TODO We will not need this assertion after we add support for multiple analyses + assert dataFrameAnalyses.size() == 1; + + AnalyticsProcessConfig processConfig = new AnalyticsProcessConfig(dataSummary.rows, dataSummary.cols, + new ByteSizeValue(1, ByteSizeUnit.GB), 1, dataFrameAnalyses.get(0)); + return processConfig; } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java index f6be7a4e78e36..a09ea7991297d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java @@ -14,7 +14,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.xpack.ml.dataframe.DataFrameDataExtractor; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor; import java.util.ArrayList; import java.util.Iterator; @@ -76,12 +76,12 @@ public void process(AnalyticsProcess process) { currentDataFrameRows = null; } } catch (Exception e) { - LOGGER.warn("Error processing analytics result", e); + LOGGER.warn("Error processing dataframe result", e); } } } catch (Exception e) { - LOGGER.error("Error parsing analytics output", e); + LOGGER.error("Error parsing dataframe output", e); } finally { completionLatch.countDown(); process.consumeAndCloseOutputStream(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java index 8148a431f0a67..e05663ab055e0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java @@ -62,7 +62,7 @@ public AnalyticsProcess createAnalyticsProcess(String jobId, AnalyticsProcessCon try { IOUtils.close(analyticsProcess); } catch (IOException ioe) { - LOGGER.error("Can't close analytics", ioe); + LOGGER.error("Can't close dataframe", ioe); } throw e; } @@ -76,7 +76,7 @@ private void createNativeProcess(String jobId, AnalyticsProcessConfig analyticsP analyticsBuilder.build(); processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); } catch (IOException e) { - String msg = "Failed to launch analytics for job " + jobId; + String msg = "Failed to launch dataframe for job " + jobId; LOGGER.error(msg); throw ExceptionsHelper.serverError(msg, e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java index dc9d77cd68784..06b377fd780d9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java @@ -122,10 +122,10 @@ public void writeHeader() throws IOException { /** * Tokenize the field that has been configured for categorization, and store the resulting list of tokens in CSV - * format in the appropriate field of the record to be sent to the analytics. + * format in the appropriate field of the record to be sent to the dataframe. * @param categorizationAnalyzer The analyzer to use to convert the categorization field to a list of tokens * @param categorizationFieldValue The value of the categorization field to be tokenized - * @param record The record to be sent to the analytics + * @param record The record to be sent to the dataframe */ protected void tokenizeForCategorization(CategorizationAnalyzer categorizationAnalyzer, String categorizationFieldValue, String[] record) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..6f2c89fd09c53 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.dataframe; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; + +public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { + + public RestGetDataFrameAnalyticsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics", this); + controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/{" + + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_get_data_frame_analytics_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetDataFrameAnalyticsAction.Request request = new GetDataFrameAnalyticsAction.Request(); + String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName()); + if (Strings.isNullOrEmpty(id) == false) { + request.setResourceId(id); + } + if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) { + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + } + + return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java new file mode 100644 index 0000000000000..a2d2b1ca48e27 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsStatsAction.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.dataframe; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; + +public class RestGetDataFrameAnalyticsStatsAction extends BaseRestHandler { + + public RestGetDataFrameAnalyticsStatsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/_stats", this); + controller.registerHandler(RestRequest.Method.GET, MachineLearning.BASE_PATH + "data_frame/analytics/{" + + DataFrameAnalyticsConfig.ID.getPreferredName() + "}/_stats", this); + } + + @Override + public String getName() { + return "xpack_ml_get_data_frame_analytics_stats_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName()); + GetDataFrameAnalyticsStatsAction.Request request = new GetDataFrameAnalyticsStatsAction.Request(); + if (Strings.isNullOrEmpty(id) == false) { + request.setId(id); + } + if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) { + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + } + + return channel -> client.execute(GetDataFrameAnalyticsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..e2422c6cdeba9 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestPutDataFrameAnalyticsAction.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.dataframe; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; + +public class RestPutDataFrameAnalyticsAction extends BaseRestHandler { + + public RestPutDataFrameAnalyticsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, MachineLearning.BASE_PATH + "data_frame/analytics/{" + + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_put_data_frame_analytics_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName()); + XContentParser parser = restRequest.contentParser(); + PutDataFrameAnalyticsAction.Request putRequest = PutDataFrameAnalyticsAction.Request.parseRequest(id, parser); + putRequest.timeout(restRequest.paramAsTime("timeout", putRequest.timeout())); + + return channel -> client.execute(PutDataFrameAnalyticsAction.INSTANCE, putRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/analytics/RestRunAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java similarity index 50% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/analytics/RestRunAnalyticsAction.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java index feadce6ff302d..035822429cc29 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/analytics/RestRunAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestStartDataFrameAnalyticsAction.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.rest.analytics; +package org.elasticsearch.xpack.ml.rest.dataframe; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; @@ -11,28 +11,30 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.ml.action.RunAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; -public class RestRunAnalyticsAction extends BaseRestHandler { +public class RestStartDataFrameAnalyticsAction extends BaseRestHandler { - public RestRunAnalyticsAction(Settings settings, RestController controller) { + public RestStartDataFrameAnalyticsAction(Settings settings, RestController controller) { super(settings); - controller.registerHandler(RestRequest.Method.POST, MachineLearning.BASE_PATH + "analytics/{index}/_run", this); + controller.registerHandler(RestRequest.Method.POST, MachineLearning.BASE_PATH + "data_frame/analytics/{" + + DataFrameAnalyticsConfig.ID.getPreferredName() + "}/_start", this); } @Override public String getName() { - return "xpack_ml_run_analytics_action"; + return "xpack_ml_start_data_frame_analytics_action"; } @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - RunAnalyticsAction.Request request = new RunAnalyticsAction.Request(restRequest.param("index")); - return channel -> { - client.execute(RunAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); - }; + String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName()); + StartDataFrameAnalyticsAction.Request request = new StartDataFrameAnalyticsAction.Request(id); + + return channel -> client.execute(StartDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java similarity index 99% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactoryTests.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java index 82c492f579e10..b64e297585373 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DataFrameDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.ml.dataframe; +package org.elasticsearch.xpack.ml.dataframe.extractor; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.fieldcaps.FieldCapabilities; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java index 1d38049ff23ce..15d2e012080e1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java @@ -16,7 +16,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ml.dataframe.DataFrameDataExtractor; +import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor; import org.junit.Before; import org.mockito.ArgumentCaptor; From 022799610dd1f7525ffb5264b39a9132605bcd80 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 28 Jan 2019 16:06:54 +0200 Subject: [PATCH 02/12] Better top level field name for the GET response --- .../xpack/core/ml/action/GetDataFrameAnalyticsAction.java | 4 ++-- .../xpack/ml/action/TransportGetDataFrameAnalyticsAction.java | 2 +- .../ml/action/TransportGetDataFrameAnalyticsStatsAction.java | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java index 0703e17f86e9b..71ebdc73e77ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java @@ -25,7 +25,7 @@ private GetDataFrameAnalyticsAction() { @Override public Response newResponse() { - return new Response(new QueryPage<>(Collections.emptyList(), 0, Response.CONFIGS)); + return new Response(new QueryPage<>(Collections.emptyList(), 0, Response.RESULTS_FIELD)); } public static class Request extends AbstractGetResourcesRequest { @@ -38,7 +38,7 @@ public String getResourceIdField() { public static class Response extends AbstractGetResourcesResponse { - public static final ParseField CONFIGS = new ParseField("configs"); + public static final ParseField RESULTS_FIELD = new ParseField("data_frame_analytics"); public Response(QueryPage analytics) { super(analytics); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java index 7b0697895c3c0..a2edc26b3ce88 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java @@ -31,7 +31,7 @@ public TransportGetDataFrameAnalyticsAction(TransportService transportService, A @Override protected ParseField getResultsField() { - return GetDataFrameAnalyticsAction.Response.CONFIGS; + return GetDataFrameAnalyticsAction.Response.RESULTS_FIELD; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java index 218e07bd3757f..3d4c935929624 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.threadpool.ThreadPool; @@ -66,7 +65,7 @@ protected void masterOperation(GetDataFrameAnalyticsStatsAction.Request request, List stats = new ArrayList(response.getResources().results().size()); response.getResources().results().forEach(c -> stats.add(buildStats(c.getId(), tasks, state))); listener.onResponse(new GetDataFrameAnalyticsStatsAction.Response(new QueryPage<>(stats, stats.size(), - new ParseField("stats")))); + GetDataFrameAnalyticsAction.Response.RESULTS_FIELD))); }, listener::onFailure ); From bcc458617e039bbcd2daaaa780acb8e97e2fb318 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 29 Jan 2019 11:04:02 +0200 Subject: [PATCH 03/12] Remove RunAnalyticsAction --- .../xpack/core/XPackClientPlugin.java | 9 +- .../core/ml/action/RunAnalyticsAction.java | 109 ------------------ 2 files changed, 7 insertions(+), 111 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 77ac71c0cf94d..c765b0390bf80 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -87,6 +87,8 @@ import org.elasticsearch.xpack.core.ml.action.GetCalendarEventsAction; import org.elasticsearch.xpack.core.ml.action.GetCalendarsAction; import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; @@ -105,11 +107,11 @@ import org.elasticsearch.xpack.core.ml.action.PostDataAction; import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction; import org.elasticsearch.xpack.core.ml.action.PutCalendarAction; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; import org.elasticsearch.xpack.core.ml.action.PutFilterAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.RunAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.SetUpgradeModeAction; import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; @@ -295,8 +297,11 @@ public List> getClientActions() { PostCalendarEventsAction.INSTANCE, PersistJobAction.INSTANCE, FindFileStructureAction.INSTANCE, - RunAnalyticsAction.INSTANCE, SetUpgradeModeAction.INSTANCE, + PutDataFrameAnalyticsAction.INSTANCE, + GetDataFrameAnalyticsAction.INSTANCE, + GetDataFrameAnalyticsStatsAction.INSTANCE, + StartDataFrameAnalyticsAction.INSTANCE, // security ClearRealmCacheAction.INSTANCE, ClearRolesCacheAction.INSTANCE, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java deleted file mode 100644 index 5d25e68e005f8..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/RunAnalyticsAction.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.ml.action; - -import org.elasticsearch.action.Action; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Objects; - -public class RunAnalyticsAction extends Action { - - public static final RunAnalyticsAction INSTANCE = new RunAnalyticsAction(); - public static final String NAME = "cluster:admin/xpack/ml/dataframe/run"; - - private RunAnalyticsAction() { - super(NAME); - } - - @Override - public AcknowledgedResponse newResponse() { - return new AcknowledgedResponse(); - } - - public static class Request extends ActionRequest implements ToXContentObject { - - private String index; - - public Request(String index) { - this.index = index; - } - - public Request(StreamInput in) throws IOException { - readFrom(in); - } - - public Request() { - } - - public String getIndex() { - return index; - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - @Override - public void readFrom(StreamInput in) throws IOException { - super.readFrom(in); - index = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(index); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("index", index); - return builder; - } - - @Override - public int hashCode() { - return Objects.hash(index); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || obj.getClass() != getClass()) { - return false; - } - RunAnalyticsAction.Request other = (RunAnalyticsAction.Request) obj; - return Objects.equals(index, other.index); - } - - @Override - public String toString() { - return Strings.toString(this); - } - } - - static class RequestBuilder extends ActionRequestBuilder { - - RequestBuilder(ElasticsearchClient client, RunAnalyticsAction action) { - super(client, action, new Request()); - } - } - -} From 2acdedc551713a102dc7d6bbd7139ef398811513 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 29 Jan 2019 11:34:34 +0200 Subject: [PATCH 04/12] Use analytics id in data extractor --- .../xpack/core/ml/job/results/ReservedFieldNames.java | 9 +++++++++ .../TransportStartDataFrameAnalyticsAction.java | 4 ++-- .../xpack/ml/dataframe/DataFrameAnalyticsManager.java | 3 ++- .../dataframe/extractor/DataFrameDataExtractor.java | 2 +- .../extractor/DataFrameDataExtractorFactory.java | 11 +++++++---- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index 333b87b0c294f..62e9f78e1c826 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -8,6 +8,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DelayedDataCheckConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -256,6 +257,14 @@ public final class ReservedFieldNames { ChunkingConfig.MODE_FIELD.getPreferredName(), ChunkingConfig.TIME_SPAN_FIELD.getPreferredName(), + DataFrameAnalyticsConfig.ID.getPreferredName(), + DataFrameAnalyticsConfig.SOURCE.getPreferredName(), + DataFrameAnalyticsConfig.DEST.getPreferredName(), + DataFrameAnalyticsConfig.ANALYSES.getPreferredName(), + "outlier_detection", + "method", + "number_neighbours", + ElasticsearchMappings.CONFIG_TYPE }; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index f100e8f91f26b..fe80ea9474483 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -125,8 +125,8 @@ public void onFailure(Exception e) { // Validate config ActionListener configListener = ActionListener.wrap( - config -> DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getSource(), ActionListener.wrap( - dataExtractorFactory -> validatedConfigListener.onResponse(config), listener::onFailure)), + config -> DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getId(), config.getSource(), + ActionListener.wrap(dataExtractorFactory -> validatedConfigListener.onResponse(config), listener::onFailure)), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index 18f0bb647c245..72b1319689820 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -142,7 +142,8 @@ private void startAnalytics(DataFrameAnalyticsTask task, DataFrameAnalyticsConfi // TODO This could fail with errors. In that case we get stuck with the copied index. // We could delete the index in case of failure or we could try building the factory before reindexing // to catch the error early on. - DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getDest(), dataExtractorFactoryListener); + DataFrameDataExtractorFactory.create(client, Collections.emptyMap(), config.getId(), config.getDest(), + dataExtractorFactoryListener); } private void createDestinationIndex(String sourceIndex, String destinationIndex, ActionListener listener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java index 6071c0b111d02..055a5e7d8dd64 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java @@ -83,7 +83,7 @@ public Optional> next() throws IOException { } protected List initScroll() throws IOException { - LOGGER.debug("[{}] Initializing scroll", "analytics"); + LOGGER.debug("[{}] Initializing scroll", context.jobId); SearchResponse searchResponse = executeSearchRequest(buildSearchRequest()); LOGGER.debug("[{}] Search response was obtained", context.jobId); return processSearchResponse(searchResponse); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java index a3d6b683e3514..e80adf37f20d6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java @@ -54,18 +54,20 @@ public class DataFrameDataExtractorFactory { } private final Client client; + private final String analyticsId; private final String index; private final ExtractedFields extractedFields; - private DataFrameDataExtractorFactory(Client client, String index, ExtractedFields extractedFields) { + private DataFrameDataExtractorFactory(Client client, String analyticsId, String index, ExtractedFields extractedFields) { this.client = Objects.requireNonNull(client); + this.analyticsId = Objects.requireNonNull(analyticsId); this.index = Objects.requireNonNull(index); this.extractedFields = Objects.requireNonNull(extractedFields); } public DataFrameDataExtractor newExtractor(boolean includeSource) { DataFrameDataExtractorContext context = new DataFrameDataExtractorContext( - "ml-dataframe-" + index, + analyticsId, extractedFields, Arrays.asList(index), QueryBuilders.matchAllQuery(), @@ -76,13 +78,14 @@ public DataFrameDataExtractor newExtractor(boolean includeSource) { return new DataFrameDataExtractor(client, context); } - public static void create(Client client, Map headers, String index, + public static void create(Client client, Map headers, String analyticsId, String index, ActionListener listener) { // Step 2. Contruct the factory and notify listener ActionListener fieldCapabilitiesHandler = ActionListener.wrap( fieldCapabilitiesResponse -> { - listener.onResponse(new DataFrameDataExtractorFactory(client, index, detectExtractedFields(fieldCapabilitiesResponse))); + listener.onResponse(new DataFrameDataExtractorFactory(client, analyticsId, index, + detectExtractedFields(fieldCapabilitiesResponse))); }, e -> { if (e instanceof IndexNotFoundException) { listener.onFailure(new ResourceNotFoundException("cannot retrieve data because index " From 3391ce5668312ce590f86221654776284922dbb0 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 29 Jan 2019 12:26:04 +0200 Subject: [PATCH 05/12] Better wording where dataframe was used --- .../xpack/core/ml/process/writer/RecordWriter.java | 2 +- .../xpack/ml/dataframe/process/AnalyticsProcess.java | 2 +- .../xpack/ml/dataframe/process/AnalyticsProcessManager.java | 2 +- .../xpack/ml/dataframe/process/AnalyticsResultProcessor.java | 4 ++-- .../ml/dataframe/process/NativeAnalyticsProcessFactory.java | 4 ++-- .../autodetect/writer/AbstractDataToProcessWriter.java | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java index 27468a2757858..2d4c636172eca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/process/writer/RecordWriter.java @@ -10,7 +10,7 @@ /** * Interface for classes that write arrays of strings to the - * Ml dataframe processes. + * Ml data frame analytics processes. */ public interface RecordWriter { /** diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java index a96bb593346b5..c5e361c3e1215 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcess.java @@ -20,7 +20,7 @@ public interface AnalyticsProcess extends NativeProcess { void writeEndOfDataMessage() throws IOException; /** - * @return stream of dataframe results. + * @return stream of data frame analytics results. */ Iterator readAnalyticsResults(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java index fe98d498d0039..b0512f47af2d5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java @@ -128,7 +128,7 @@ private AnalyticsProcess createProcess(String jobId, AnalyticsProcessConfig anal ExecutorService executorService = threadPool.executor(MachineLearning.AUTODETECT_THREAD_POOL_NAME); AnalyticsProcess process = processFactory.createAnalyticsProcess(jobId, analyticsProcessConfig, executorService); if (process.isProcessAlive() == false) { - throw ExceptionsHelper.serverError("Failed to start dataframe process"); + throw ExceptionsHelper.serverError("Failed to start data frame analytics process"); } return process; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java index a09ea7991297d..cd6864f049740 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java @@ -76,12 +76,12 @@ public void process(AnalyticsProcess process) { currentDataFrameRows = null; } } catch (Exception e) { - LOGGER.warn("Error processing dataframe result", e); + LOGGER.warn("Error processing data frame analytics result", e); } } } catch (Exception e) { - LOGGER.error("Error parsing dataframe output", e); + LOGGER.error("Error parsing data frame analytics output", e); } finally { completionLatch.countDown(); process.consumeAndCloseOutputStream(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java index e05663ab055e0..14743b93dc424 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/NativeAnalyticsProcessFactory.java @@ -62,7 +62,7 @@ public AnalyticsProcess createAnalyticsProcess(String jobId, AnalyticsProcessCon try { IOUtils.close(analyticsProcess); } catch (IOException ioe) { - LOGGER.error("Can't close dataframe", ioe); + LOGGER.error("Can't close data frame analytics process", ioe); } throw e; } @@ -76,7 +76,7 @@ private void createNativeProcess(String jobId, AnalyticsProcessConfig analyticsP analyticsBuilder.build(); processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); } catch (IOException e) { - String msg = "Failed to launch dataframe for job " + jobId; + String msg = "Failed to launch data frame analytics process for job " + jobId; LOGGER.error(msg); throw ExceptionsHelper.serverError(msg, e); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java index 06b377fd780d9..799954619b315 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java @@ -122,10 +122,10 @@ public void writeHeader() throws IOException { /** * Tokenize the field that has been configured for categorization, and store the resulting list of tokens in CSV - * format in the appropriate field of the record to be sent to the dataframe. + * format in the appropriate field of the record to be sent to the process. * @param categorizationAnalyzer The analyzer to use to convert the categorization field to a list of tokens * @param categorizationFieldValue The value of the categorization field to be tokenized - * @param record The record to be sent to the dataframe + * @param record The record to be sent to the process */ protected void tokenizeForCategorization(CategorizationAnalyzer categorizationAnalyzer, String categorizationFieldValue, String[] record) { From 71dc98687b01f774637d0f0bec00ad7996a5bc49 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 29 Jan 2019 13:00:13 +0200 Subject: [PATCH 06/12] Remove double call to finishHandler and add TODO to deal with such failure --- .../xpack/ml/dataframe/process/AnalyticsProcessManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java index b0512f47af2d5..168662e5cf3c0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java @@ -70,7 +70,7 @@ private void processData(String jobId, DataFrameDataExtractor dataExtractor, Ana LOGGER.info("[{}] Result processor has completed", jobId); } catch (IOException e) { LOGGER.error(new ParameterizedMessage("[{}] Error writing data to the process", jobId), e); - finishHandler.accept(e); + // TODO Handle this failure by setting the task state to FAILED } finally { LOGGER.info("[{}] Closing process", jobId); try { From a7f8e286e320907a7a78b48dd3dc0f3d4cb1768a Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 31 Jan 2019 15:44:24 +0200 Subject: [PATCH 07/12] Delete API plus first batch of YAML tests --- .../DeleteDataFrameAnalyticsAction.java | 81 ++++ .../action/PutDataFrameAnalyticsAction.java | 58 ++- .../ml/dataframe/DataFrameAnalysisConfig.java | 4 +- .../dataframe/DataFrameAnalyticsConfig.java | 2 +- .../xpack/core/ml/utils/ExceptionsHelper.java | 8 + ...tDataFrameAnalyticsActionRequestTests.java | 42 ++ .../DataFrameAnalyticsConfigTests.java | 13 +- .../integration/MlRestTestStateCleaner.java | 17 + .../xpack/ml/MachineLearning.java | 5 + .../AbstractTransportGetResourcesAction.java | 14 +- ...ansportDeleteDataFrameAnalyticsAction.java | 96 +++++ .../TransportGetDataFrameAnalyticsAction.java | 3 +- ...sportGetDataFrameAnalyticsStatsAction.java | 1 - .../TransportPutDataFrameAnalyticsAction.java | 2 +- .../DataFrameDataExtractorFactory.java | 6 +- .../DataFrameAnalyticsConfigProvider.java | 12 +- .../RestDeleteDataFrameAnalyticsAction.java | 39 ++ .../DataFrameDataExtractorFactoryTests.java | 22 +- .../api/ml.delete_data_frame_analytics.json | 17 + .../api/ml.get_data_frame_analytics.json | 29 ++ .../ml.get_data_frame_analytics_stats.json | 29 ++ .../api/ml.put_data_frame_analytics.json | 20 + .../api/ml.start_data_frame_analytics.json | 16 + .../test/ml/data_frame_analytics_crud.yml | 370 ++++++++++++++++++ .../test/ml/start_data_frame_analytics.yml | 46 +++ 25 files changed, 919 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..27ee6b0a97139 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/DeleteDataFrameAnalyticsAction.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteDataFrameAnalyticsAction extends Action { + + public static final DeleteDataFrameAnalyticsAction INSTANCE = new DeleteDataFrameAnalyticsAction(); + public static final String NAME = "cluster:admin/xpack/ml/data_frame/analytics/delete"; + + private DeleteDataFrameAnalyticsAction() { + super(NAME); + } + + @Override + public AcknowledgedResponse newResponse() { + return new AcknowledgedResponse(); + } + + public static class Request extends AcknowledgedRequest implements ToXContentFragment { + + private String id; + + public Request() {} + + public Request(String id) { + this.id = ExceptionsHelper.requireNonNull(id, DataFrameAnalyticsConfig.ID); + } + + public String getId() { + return id; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(DataFrameAnalyticsConfig.ID.getPreferredName(), id); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeleteDataFrameAnalyticsAction.Request request = (DeleteDataFrameAnalyticsAction.Request) o; + return Objects.equals(id, request.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, DeleteDataFrameAnalyticsAction action) { + super(client, action, new Request()); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java index ec21024783e1e..e447aa70109e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsAction.java @@ -7,11 +7,13 @@ import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -58,6 +60,18 @@ public Request(DataFrameAnalyticsConfig config) { this.config = config; } + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + config = new DataFrameAnalyticsConfig(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + config.writeTo(out); + } + public DataFrameAnalyticsConfig getConfig() { return config; } @@ -87,13 +101,45 @@ public int hashCode() { } } - public static class Response extends AcknowledgedResponse { - public Response() { - super(); + public static class Response extends ActionResponse implements ToXContentObject { + + private DataFrameAnalyticsConfig config; + + public Response(DataFrameAnalyticsConfig config) { + this.config = config; + } + + Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + config = new DataFrameAnalyticsConfig(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + config.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + config.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(config, response.config); } - public Response(boolean acknowledged) { - super(acknowledged); + @Override + public int hashCode() { + return Objects.hash(config); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java index 9c7ce7a9ce31d..acdb21b44dadf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalysisConfig.java @@ -20,16 +20,16 @@ public class DataFrameAnalysisConfig implements ToXContentObject, Writeable { public static ContextParser parser() { - return (p, c) -> new DataFrameAnalysisConfig(p.map()); + return (p, c) -> new DataFrameAnalysisConfig(p.mapOrdered()); } private final Map config; public DataFrameAnalysisConfig(Map config) { + this.config = Objects.requireNonNull(config); if (config.size() != 1) { throw ExceptionsHelper.badRequestException("A data frame analysis must specify exactly one analysis type"); } - this.config = Objects.requireNonNull(config); } public DataFrameAnalysisConfig(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 9e32d444b58fe..028e4347ba8f0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -55,7 +55,7 @@ public DataFrameAnalyticsConfig(String id, String source, String dest, List 1) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java index c847b679a7493..320eace983590 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ExceptionsHelper.java @@ -35,6 +35,14 @@ public static ResourceAlreadyExistsException datafeedAlreadyExists(String datafe return new ResourceAlreadyExistsException(Messages.getMessage(Messages.DATAFEED_ID_ALREADY_TAKEN, datafeedId)); } + public static ResourceNotFoundException missingDataFrameAnalytics(String id) { + return new ResourceNotFoundException("No known data frame analytics with id [{}]", id); + } + + public static ResourceAlreadyExistsException dataFrameAnalyticsAlreadyExists(String id) { + return new ResourceAlreadyExistsException("A data frame analytics with id [{}] already exists", id); + } + public static ElasticsearchException serverError(String msg) { return new ElasticsearchException(msg); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java new file mode 100644 index 0000000000000..2d899b7fb2d44 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionRequestTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction.Request; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; +import org.junit.Before; + +public class PutDataFrameAnalyticsActionRequestTests extends AbstractStreamableXContentTestCase { + + private String id; + + @Before + public void setUpId() { + id = DataFrameAnalyticsConfigTests.randomValidId(); + } + + @Override + protected Request createTestInstance() { + return new Request(DataFrameAnalyticsConfigTests.createRandom(id)); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request doParseInstance(XContentParser parser) { + return Request.parseRequest(id, parser); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java index 9e983666f0334..dfb3e8ef36546 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -23,11 +23,7 @@ protected DataFrameAnalyticsConfig doParseInstance(XContentParser parser) throws @Override protected DataFrameAnalyticsConfig createTestInstance() { - String id = randomValidId(); - String source = randomAlphaOfLength(10); - String dest = randomAlphaOfLength(10); - List analyses = Collections.singletonList(DataFrameAnalysisConfigTests.randomConfig()); - return new DataFrameAnalyticsConfig(id, source, dest, analyses); + return createRandom(randomValidId()); } @Override @@ -35,6 +31,13 @@ protected Writeable.Reader instanceReader() { return DataFrameAnalyticsConfig::new; } + public static DataFrameAnalyticsConfig createRandom(String id) { + String source = randomAlphaOfLength(10); + String dest = randomAlphaOfLength(10); + List analyses = Collections.singletonList(DataFrameAnalysisConfigTests.randomConfig()); + return new DataFrameAnalyticsConfig(id, source, dest, analyses); + } + public static String randomValidId() { CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray()); return generator.ofCodePointsLength(random(), 10, 10); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java index 46d7c5b9e43da..00edb6bb6df2c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java @@ -29,6 +29,7 @@ public MlRestTestStateCleaner(Logger logger, RestClient adminClient) { public void clearMlMetadata() throws IOException { deleteAllDatafeeds(); deleteAllJobs(); + deleteAllDataFrameAnalytics(); // indices will be deleted by the ESRestTestCase class } @@ -91,4 +92,20 @@ private void deleteAllJobs() throws IOException { adminClient.performRequest(new Request("DELETE", "/_ml/anomaly_detectors/" + jobId)); } } + + private void deleteAllDataFrameAnalytics() throws IOException { + final Request analyticsRequest = new Request("GET", "/_ml/data_frame/analytics?size=10000"); + analyticsRequest.addParameter("filter_path", "data_frame_analytics"); + final Response analyticsResponse = adminClient.performRequest(analyticsRequest); + List> analytics = (List>) XContentMapValues.extractValue( + "data_frame_analytics", ESRestTestCase.entityAsMap(analyticsResponse)); + if (analytics == null) { + return; + } + + for (Map config : analytics) { + String id = (String) config.get("id"); + adminClient.performRequest(new Request("DELETE", "/_ml/data_frame/analytics/" + id)); + } + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 833436d7ed7ee..bf9893e7f74d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -61,6 +61,7 @@ import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarAction; import org.elasticsearch.xpack.core.ml.action.DeleteCalendarEventAction; +import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.core.ml.action.DeleteExpiredDataAction; import org.elasticsearch.xpack.core.ml.action.DeleteFilterAction; @@ -121,6 +122,7 @@ import org.elasticsearch.xpack.ml.action.TransportCloseJobAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction; import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarEventAction; +import org.elasticsearch.xpack.ml.action.TransportDeleteDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.action.TransportDeleteDatafeedAction; import org.elasticsearch.xpack.ml.action.TransportDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.action.TransportDeleteFilterAction; @@ -222,6 +224,7 @@ import org.elasticsearch.xpack.ml.rest.datafeeds.RestStartDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestStopDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestUpdateDatafeedAction; +import org.elasticsearch.xpack.ml.rest.dataframe.RestDeleteDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestGetDataFrameAnalyticsStatsAction; import org.elasticsearch.xpack.ml.rest.dataframe.RestPutDataFrameAnalyticsAction; @@ -582,6 +585,7 @@ public List getRestHandlers(Settings settings, RestController restC new RestGetDataFrameAnalyticsAction(settings, restController), new RestGetDataFrameAnalyticsStatsAction(settings, restController), new RestPutDataFrameAnalyticsAction(settings, restController), + new RestDeleteDataFrameAnalyticsAction(settings, restController), new RestStartDataFrameAnalyticsAction(settings, restController) ); } @@ -645,6 +649,7 @@ public List getRestHandlers(Settings settings, RestController restC new ActionHandler<>(GetDataFrameAnalyticsAction.INSTANCE, TransportGetDataFrameAnalyticsAction.class), new ActionHandler<>(GetDataFrameAnalyticsStatsAction.INSTANCE, TransportGetDataFrameAnalyticsStatsAction.class), new ActionHandler<>(PutDataFrameAnalyticsAction.INSTANCE, TransportPutDataFrameAnalyticsAction.class), + new ActionHandler<>(DeleteDataFrameAnalyticsAction.INSTANCE, TransportDeleteDataFrameAnalyticsAction.class), new ActionHandler<>(StartDataFrameAnalyticsAction.INSTANCE, TransportStartDataFrameAnalyticsAction.class) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java index 616d22116312f..56fa331230f03 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/AbstractTransportGetResourcesAction.java @@ -49,6 +49,8 @@ public abstract class AbstractTransportGetResourcesAction> extends HandledTransportAction { + private static final String ALL = "_all"; + private Client client; protected AbstractTransportGetResourcesAction(String actionName, TransportService transportService, ActionFilters actionFilters, @@ -83,7 +85,7 @@ public void onResponse(SearchResponse response) { } } - if (docs.isEmpty() && Regex.isSimpleMatchPattern(request.getResourceId()) == false) { + if (docs.isEmpty() && isConcreteMatch(request.getResourceId())) { listener.onFailure(notFoundException(request.getResourceId())); } else { listener.onResponse(new QueryPage<>(docs, docs.size(), getResultsField())); @@ -101,7 +103,7 @@ public void onFailure(Exception e) { private QueryBuilder buildQuery(AbstractGetResourcesRequest request) { BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - if (Strings.isNullOrEmpty(request.getResourceId()) == false) { + if (isMatchAll(request.getResourceId()) == false) { boolQuery.filter(QueryBuilders.wildcardQuery(request.getResourceIdField(), request.getResourceId())); } QueryBuilder additionalQuery = additionalQuery(); @@ -111,6 +113,14 @@ private QueryBuilder buildQuery(AbstractGetResourcesRequest request) { return boolQuery.hasClauses() ? boolQuery : QueryBuilders.matchAllQuery(); } + private static boolean isMatchAll(String resourceId) { + return Strings.isNullOrEmpty(resourceId) || ALL.equals(resourceId) || Regex.isMatchAllPattern(resourceId); + } + + private static boolean isConcreteMatch(String resourceId) { + return isMatchAll(resourceId) == false && Regex.isSimpleMatchPattern(resourceId) == false; + } + @Nullable protected QueryBuilder additionalQuery() { return null; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..c14f8bef92c0e --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteDataFrameAnalyticsAction.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ml.MlTasks; +import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; + +/** + * The action is a master node action to ensure it reads an up-to-date cluster + * state in order to determine whether there is a persistent task for the analytics + * to delete. + */ +public class TransportDeleteDataFrameAnalyticsAction + extends TransportMasterNodeAction { + + private final Client client; + + @Inject + public TransportDeleteDataFrameAnalyticsAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + super(DeleteDataFrameAnalyticsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, + DeleteDataFrameAnalyticsAction.Request::new); + this.client = client; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse newResponse() { + return new AcknowledgedResponse(); + } + + @Override + protected void masterOperation(DeleteDataFrameAnalyticsAction.Request request, ClusterState state, + ActionListener listener) { + PersistentTasksCustomMetaData tasks = state.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); + DataFrameAnalyticsState taskState = MlTasks.getDataFrameAnalyticsState(request.getId(), tasks); + if (taskState != DataFrameAnalyticsState.STOPPED) { + listener.onFailure(ExceptionsHelper.conflictStatusException("Cannot delete data frame analytics [{}] while its status is [{}]", + request.getId(), taskState)); + return; + } + + DeleteRequest deleteRequest = new DeleteRequest(AnomalyDetectorsIndex.configIndexName()); + deleteRequest.id(DataFrameAnalyticsConfig.documentId(request.getId())); + deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + executeAsyncWithOrigin(client, ML_ORIGIN, DeleteAction.INSTANCE, deleteRequest, ActionListener.wrap( + deleteResponse -> { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + listener.onFailure(ExceptionsHelper.missingDataFrameAnalytics(request.getId())); + return; + } + assert deleteResponse.getResult() == DocWriteResponse.Result.DELETED; + listener.onResponse(new AcknowledgedResponse(true)); + }, + listener::onFailure + )); + } + + @Override + protected ClusterBlockException checkBlock(DeleteDataFrameAnalyticsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java index a2edc26b3ce88..02083b6c7d45e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; public class TransportGetDataFrameAnalyticsAction extends AbstractTransportGetResourcesAction { @@ -46,7 +47,7 @@ protected DataFrameAnalyticsConfig parse(XContentParser parser) { @Override protected ResourceNotFoundException notFoundException(String resourceId) { - return new ResourceNotFoundException("No known data frame analytics with id [{}]", resourceId); + return ExceptionsHelper.missingDataFrameAnalytics(resourceId); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java index 3d4c935929624..1a30200668d93 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -74,7 +74,6 @@ protected void masterOperation(GetDataFrameAnalyticsStatsAction.Request request, getRequest.setResourceId(request.getId()); getRequest.setPageParams(request.getPageParams()); executeAsyncWithOrigin(client, ML_ORIGIN, GetDataFrameAnalyticsAction.INSTANCE, getRequest, getResponseListener); - } private GetDataFrameAnalyticsStatsAction.Response.Stats buildStats(String concreteAnalyticsId, PersistentTasksCustomMetaData tasks, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index 33e8d8e6c6bfc..e96ca02ce4f89 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -49,7 +49,7 @@ protected void doExecute(Task task, PutDataFrameAnalyticsAction.Request request, validateConfig(request.getConfig()); configProvider.put(request.getConfig(), ActionListener.wrap( - indexResponse -> listener.onResponse(new PutDataFrameAnalyticsAction.Response(true)), + indexResponse -> listener.onResponse(new PutDataFrameAnalyticsAction.Response(request.getConfig())), listener::onFailure )); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java index e80adf37f20d6..a9b225dee77ac 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java @@ -85,7 +85,7 @@ public static void create(Client client, Map headers, String ana ActionListener fieldCapabilitiesHandler = ActionListener.wrap( fieldCapabilitiesResponse -> { listener.onResponse(new DataFrameDataExtractorFactory(client, analyticsId, index, - detectExtractedFields(fieldCapabilitiesResponse))); + detectExtractedFields(index, fieldCapabilitiesResponse))); }, e -> { if (e instanceof IndexNotFoundException) { listener.onFailure(new ResourceNotFoundException("cannot retrieve data because index " @@ -108,7 +108,7 @@ public static void create(Client client, Map headers, String ana } // Visible for testing - static ExtractedFields detectExtractedFields(FieldCapabilitiesResponse fieldCapabilitiesResponse) { + static ExtractedFields detectExtractedFields(String index, FieldCapabilitiesResponse fieldCapabilitiesResponse) { Set fields = fieldCapabilitiesResponse.get().keySet(); fields.removeAll(IGNORE_FIELDS); removeFieldsWithIncompatibleTypes(fields, fieldCapabilitiesResponse); @@ -118,7 +118,7 @@ static ExtractedFields detectExtractedFields(FieldCapabilitiesResponse fieldCapa ExtractedFields extractedFields = ExtractedFields.build(sortedFields, Collections.emptySet(), fieldCapabilitiesResponse) .filterFields(ExtractedField.ExtractionMethod.DOC_VALUE); if (extractedFields.getAllFields().isEmpty()) { - throw ExceptionsHelper.badRequestException("No compatible fields could be detected"); + throw ExceptionsHelper.badRequestException("No compatible fields could be detected in index [{}]", index); } return extractedFields; } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java index 39c149ee02fbf..ed340d155a8fd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/persistence/DataFrameAnalyticsConfigProvider.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; @@ -57,7 +58,16 @@ public void put(DataFrameAnalyticsConfig config, ActionListener l .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(builder); - executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, listener); + executeAsyncWithOrigin(client, ML_ORIGIN, IndexAction.INSTANCE, indexRequest, ActionListener.wrap( + listener::onResponse, + e -> { + if (e instanceof VersionConflictEngineException) { + listener.onFailure(ExceptionsHelper.dataFrameAnalyticsAlreadyExists(config.getId())); + } else { + listener.onFailure(e); + } + } + )); } catch (IOException e) { listener.onFailure(new ElasticsearchParseException("Failed to serialise data frame analytics with id [" + config.getId() + "]")); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java new file mode 100644 index 0000000000000..31a9ba690a9b2 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestDeleteDataFrameAnalyticsAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.dataframe; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.ml.action.DeleteDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.ml.MachineLearning; + +import java.io.IOException; + +public class RestDeleteDataFrameAnalyticsAction extends BaseRestHandler { + + public RestDeleteDataFrameAnalyticsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, MachineLearning.BASE_PATH + "data_frame/analytics/{" + + DataFrameAnalyticsConfig.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_delete_data_frame_analytics_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String id = restRequest.param(DataFrameAnalyticsConfig.ID.getPreferredName()); + DeleteDataFrameAnalyticsAction.Request request = new DeleteDataFrameAnalyticsAction.Request(id); + return channel -> client.execute(DeleteDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java index b64e297585373..9aac76da91e95 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactoryTests.java @@ -26,11 +26,13 @@ public class DataFrameDataExtractorFactoryTests extends ESTestCase { + private static final String INDEX = "source_index"; + public void testDetectExtractedFields_GivenFloatField() { FieldCapabilitiesResponse fieldCapabilities= new MockFieldCapsResponseBuilder() .addAggregatableField("some_float", "float").build(); - ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities); + ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities); List allFields = extractedFields.getAllFields(); assertThat(allFields.size(), equalTo(1)); @@ -42,7 +44,7 @@ public void testDetectExtractedFields_GivenNumericFieldWithMultipleTypes() { .addAggregatableField("some_number", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float") .build(); - ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities); + ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities); List allFields = extractedFields.getAllFields(); assertThat(allFields.size(), equalTo(1)); @@ -54,8 +56,8 @@ public void testDetectExtractedFields_GivenNonNumericField() { .addAggregatableField("some_keyword", "keyword").build(); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities)); - assertThat(e.getMessage(), equalTo("No compatible fields could be detected")); + () -> DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities)); + assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]")); } public void testDetectExtractedFields_GivenFieldWithNumericAndNonNumericTypes() { @@ -63,8 +65,8 @@ public void testDetectExtractedFields_GivenFieldWithNumericAndNonNumericTypes() .addAggregatableField("indecisive_field", "float", "keyword").build(); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities)); - assertThat(e.getMessage(), equalTo("No compatible fields could be detected")); + () -> DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities)); + assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]")); } public void testDetectExtractedFields_GivenMultipleFields() { @@ -74,7 +76,7 @@ public void testDetectExtractedFields_GivenMultipleFields() { .addAggregatableField("some_keyword", "keyword") .build(); - ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities); + ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities); List allFields = extractedFields.getAllFields(); assertThat(allFields.size(), equalTo(2)); @@ -87,8 +89,8 @@ public void testDetectExtractedFields_GivenIgnoredField() { .addAggregatableField("_id", "float").build(); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, - () -> DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities)); - assertThat(e.getMessage(), equalTo("No compatible fields could be detected")); + () -> DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities)); + assertThat(e.getMessage(), equalTo("No compatible fields could be detected in index [source_index]")); } public void testDetectExtractedFields_ShouldSortFieldsAlphabetically() { @@ -106,7 +108,7 @@ public void testDetectExtractedFields_ShouldSortFieldsAlphabetically() { } FieldCapabilitiesResponse fieldCapabilities = mockFieldCapsResponseBuilder.build(); - ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(fieldCapabilities); + ExtractedFields extractedFields = DataFrameDataExtractorFactory.detectExtractedFields(INDEX, fieldCapabilities); List extractedFieldNames = extractedFields.getAllFields().stream().map(ExtractedField::getName) .collect(Collectors.toList()); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json new file mode 100644 index 0000000000000..a09259fabb8be --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.delete_data_frame_analytics.json @@ -0,0 +1,17 @@ +{ + "ml.delete_data_frame_analytics": { + "methods": [ "DELETE" ], + "url": { + "path": "/_ml/data_frame/analytics/{id}", + "paths": [ "/_ml/data_frame/analytics/{id}" ], + "parts": { + "id": { + "type" : "string", + "required" : true, + "description" : "The ID of the data frame analytics to delete" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json new file mode 100644 index 0000000000000..9c65661d254a4 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -0,0 +1,29 @@ +{ + "ml.get_data_frame_analytics": { + "methods": [ "GET"], + "url": { + "path": "/_ml/data_frame/analytics/{id}", + "paths": [ + "/_ml/data_frame/analytics/{id}", + "/_ml/data_frame/analytics" + ], + "parts": { + "id": { + "type": "string", + "description": "The ID of the data frame analytics to fetch" + } + }, + "params": { + "from": { + "type": "int", + "description": "skips a number of analytics" + }, + "size": { + "type": "int", + "description": "specifies a max number of analytics to get" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json new file mode 100644 index 0000000000000..d74f5880c72de --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics_stats.json @@ -0,0 +1,29 @@ +{ + "ml.get_data_frame_analytics_stats": { + "methods": [ "GET"], + "url": { + "path": "/_ml/data_frame/analytics/{id}/_stats", + "paths": [ + "/_ml/data_frame/analytics/_stats", + "/_ml/data_frame/analytics/{id}/_stats" + ], + "parts": { + "id": { + "type": "string", + "description": "The ID of the data frame analytics stats to fetch" + } + }, + "params": { + "from": { + "type": "int", + "description": "skips a number of analytics" + }, + "size": { + "type": "int", + "description": "specifies a max number of analytics to get" + } + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json new file mode 100644 index 0000000000000..1f3183920aca5 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_data_frame_analytics.json @@ -0,0 +1,20 @@ +{ + "ml.put_data_frame_analytics": { + "methods": [ "PUT" ], + "url": { + "path": "/_ml/data_frame/analytics/{id}", + "paths": [ "/_ml/data_frame/analytics/{id}" ], + "parts": { + "id": { + "type": "string", + "required": true, + "description": "The ID of the data frame analytics to create" + } + } + }, + "body": { + "description" : "The data frame analytics configuration", + "required" : true + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json new file mode 100644 index 0000000000000..b4e61b3fab125 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.start_data_frame_analytics.json @@ -0,0 +1,16 @@ +{ + "ml.start_data_frame_analytics": { + "methods": [ "POST" ], + "url": { + "path": "/_ml/data_frame/analytics/{id}/_start", + "paths": [ "/_ml/data_frame/analytics/{id}/_start" ], + "parts": { + "id": { + "type": "string", + "required": true, + "description": "The ID of the data frame analytics to start" + } + } + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml new file mode 100644 index 0000000000000..ad0160647c8fe --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -0,0 +1,370 @@ +--- +"Test get-all and get-all-stats given no analytics exist": + + - do: + ml.get_data_frame_analytics: + id: "_all" + - match: { count: 0 } + - match: { data_frame_analytics: [] } + + - do: + ml.get_data_frame_analytics: + id: "_all" + - match: { count: 0 } + - match: { data_frame_analytics: [] } + + - do: + ml.get_data_frame_analytics: + id: "*" + - match: { count: 0 } + - match: { data_frame_analytics: [] } + + - do: + ml.get_data_frame_analytics: + id: "*" + - match: { count: 0 } + - match: { data_frame_analytics: [] } + +--- +"Test put valid config with default outlier detection": + + - do: + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}] + } + - match: { id: "simple-outlier-detection" } + - match: { source: "source_index" } + - match: { dest: "dest_index" } + - match: { analyses: [{"outlier_detection":{}}] } + +--- +"Test put config with inconsistent body/param ids": + + - do: + catch: /Inconsistent id; 'body_id' specified in the body differs from 'url_id' specified as a URL argument/ + ml.put_data_frame_analytics: + id: "url_id" + body: > + { + "id": "body_id", + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}] + } + +--- +"Test put config with invalid id": + + - do: + catch: /Invalid id*/ + ml.put_data_frame_analytics: + id: "this id contains spaces" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}] + } + +--- +"Test put config with unknown top level field": + + - do: + catch: /unknown field \[unknown_field\], parser not found/ + ml.put_data_frame_analytics: + id: "unknown_field" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}], + "unknown_field": 42 + } + +--- +"Test put config with unknown field in outlier detection analysis": + + - do: + catch: /Data frame analysis \[outlier_detection\] does not support one or more provided parameters[:] \[unknown_field\]/ + ml.put_data_frame_analytics: + id: "unknown_field" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{"unknown_field": 42}}] + } + +--- +"Test put config given missing source": + + - do: + catch: /\[source\] must not be null/ + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}] + } + +--- +"Test put config given missing dest": + + - do: + catch: /\[dest\] must not be null/ + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": "source_index", + "analyses": [{"outlier_detection":{}}] + } + +--- +"Test put config given missing analyses": + + - do: + catch: /\[analyses\] must not be null/ + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": "source_index", + "dest": "dest_index" + } + +--- +"Test put config given empty analyses": + + - do: + catch: /One or more analyses are required/ + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [] + } + +--- +"Test put config given two analyses": + + - do: + catch: /Does not yet support multiple analyses/ + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": "source_index", + "dest": "dest_index", + "analyses": [{"outlier_detection":{}}, {"outlier_detection":{}}] + } + +--- +"Test get given multiple analytics": + + - do: + ml.put_data_frame_analytics: + id: "foo-1" + body: > + { + "source": "foo-1_source", + "dest": "foo-1_dest", + "analyses": [{"outlier_detection":{}}] + } + + - do: + ml.put_data_frame_analytics: + id: "foo-2" + body: > + { + "source": "foo-2_source", + "dest": "foo-2_dest", + "analyses": [{"outlier_detection":{}}] + } + - match: { id: "foo-2" } + + - do: + ml.put_data_frame_analytics: + id: "bar" + body: > + { + "source": "bar_source", + "dest": "bar_dest", + "analyses": [{"outlier_detection":{}}] + } + - match: { id: "bar" } + + - do: + ml.get_data_frame_analytics: + id: "*" + - match: { count: 3 } + - match: { data_frame_analytics.0.id: "bar" } + - match: { data_frame_analytics.1.id: "foo-1" } + - match: { data_frame_analytics.2.id: "foo-2" } + + - do: + ml.get_data_frame_analytics: + id: "foo-*" + - match: { count: 2 } + - match: { data_frame_analytics.0.id: "foo-1" } + - match: { data_frame_analytics.1.id: "foo-2" } + + - do: + ml.get_data_frame_analytics: + id: "bar" + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "bar" } + + - do: + ml.get_data_frame_analytics: + from: 1 + - match: { count: 2 } + - match: { data_frame_analytics.0.id: "foo-1" } + - match: { data_frame_analytics.1.id: "foo-2" } + + - do: + ml.get_data_frame_analytics: + size: 2 + - match: { count: 2 } + - match: { data_frame_analytics.0.id: "bar" } + - match: { data_frame_analytics.1.id: "foo-1" } + + - do: + ml.get_data_frame_analytics: + from: 1 + size: 1 + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "foo-1" } + +--- +"Test get given missing analytics": + + - do: + catch: missing + ml.get_data_frame_analytics: + id: "missing-analytics" + +--- +"Test get stats given multiple analytics": + + - do: + ml.put_data_frame_analytics: + id: "foo-1" + body: > + { + "source": "foo-1_source", + "dest": "foo-1_dest", + "analyses": [{"outlier_detection":{}}] + } + + - do: + ml.put_data_frame_analytics: + id: "foo-2" + body: > + { + "source": "foo-2_source", + "dest": "foo-2_dest", + "analyses": [{"outlier_detection":{}}] + } + - match: { id: "foo-2" } + + - do: + ml.put_data_frame_analytics: + id: "bar" + body: > + { + "source": "bar_source", + "dest": "bar_dest", + "analyses": [{"outlier_detection":{}}] + } + - match: { id: "bar" } + + - do: + ml.get_data_frame_analytics_stats: + id: "*" + - match: { count: 3 } + - match: { data_frame_analytics.0.id: "bar" } + - match: { data_frame_analytics.0.state: "stopped" } + - match: { data_frame_analytics.1.id: "foo-1" } + - match: { data_frame_analytics.1.state: "stopped" } + - match: { data_frame_analytics.2.id: "foo-2" } + - match: { data_frame_analytics.2.state: "stopped" } + + - do: + ml.get_data_frame_analytics_stats: + id: "foo-*" + - match: { count: 2 } + - match: { data_frame_analytics.0.id: "foo-1" } + - match: { data_frame_analytics.0.state: "stopped" } + - match: { data_frame_analytics.1.id: "foo-2" } + - match: { data_frame_analytics.1.state: "stopped" } + + - do: + ml.get_data_frame_analytics_stats: + id: "bar" + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "bar" } + - match: { data_frame_analytics.0.state: "stopped" } + + - do: + ml.get_data_frame_analytics_stats: + from: 2 + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "foo-2" } + - match: { data_frame_analytics.0.state: "stopped" } + + - do: + ml.get_data_frame_analytics_stats: + size: 2 + - match: { count: 2 } + - match: { data_frame_analytics.0.id: "bar" } + - match: { data_frame_analytics.0.state: "stopped" } + - match: { data_frame_analytics.1.id: "foo-1" } + - match: { data_frame_analytics.1.state: "stopped" } + + - do: + ml.get_data_frame_analytics_stats: + from: 1 + size: 1 + - match: { count: 1 } + - match: { data_frame_analytics.0.id: "foo-1" } + - match: { data_frame_analytics.0.state: "stopped" } + +--- +"Test delete given stopped config": + + - do: + ml.put_data_frame_analytics: + id: "foo" + body: > + { + "source": "source", + "dest": "dest", + "analyses": [{"outlier_detection":{}}] + } + + - do: + ml.delete_data_frame_analytics: + id: "foo" + - match: { acknowledged: true } + + - do: + catch: missing + ml.get_data_frame_analytics: + id: "foo" + +--- +"Test delete given missing config": + + - do: + catch: missing + ml.delete_data_frame_analytics: + id: "missing_config" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml new file mode 100644 index 0000000000000..36e50b0229737 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/start_data_frame_analytics.yml @@ -0,0 +1,46 @@ +--- +"Test start given missing config": + - do: + catch: missing + ml.start_data_frame_analytics: + id: "missing_config" + +--- +"Test start given missing source index": + + - do: + ml.put_data_frame_analytics: + id: "missing_index" + body: > + { + "source": "missing", + "dest": "missing-dest", + "analyses": [{"outlier_detection":{}}] + } + + - do: + catch: /cannot retrieve data because index \[missing\] does not exist/ + ml.start_data_frame_analytics: + id: "missing_index" + +--- +"Test start given source index has no compatible fields": + + - do: + indices.create: + index: empty-index + + - do: + ml.put_data_frame_analytics: + id: "foo" + body: > + { + "source": "empty-index", + "dest": "empty-index-dest", + "analyses": [{"outlier_detection":{}}] + } + + - do: + catch: /No compatible fields could be detected in index \[empty-index\]/ + ml.start_data_frame_analytics: + id: "foo" From 5e17cdd89545301b5470d7fded21e6d77ca6a7b5 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 1 Feb 2019 12:56:19 +0200 Subject: [PATCH 08/12] Add unit tests --- .../action/AbstractGetResourcesResponse.java | 2 + .../action/GetDataFrameAnalyticsAction.java | 10 +++ ...DataFrameAnalyticsActionResponseTests.java | 33 ++++++++ .../GetDataFrameAnalyticsRequestTests.java | 27 +++++++ ...rameAnalyticsStatsActionResponseTests.java | 36 +++++++++ ...etDataFrameAnalyticsStatsRequestTests.java | 26 +++++++ ...DataFrameAnalyticsActionResponseTests.java | 23 ++++++ .../StartDataFrameAnalyticsRequestTests.java | 23 ++++++ .../ml/qa/ml-with-security/build.gradle | 14 ++++ .../analyses/DataFrameAnalysesUtils.java | 13 ++-- .../dataframe/analyses/DataFrameAnalysis.java | 7 +- .../analyses/DataFrameAnalysesUtilsTests.java | 77 +++++++++++++++++++ .../analyses/OutlierDetectionTests.java | 60 +++++++++++++++ 13 files changed, 342 insertions(+), 9 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtilsTests.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetectionTests.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java index f65d4d99c6697..7f7686ff230ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/AbstractGetResourcesResponse.java @@ -24,6 +24,8 @@ public abstract class AbstractGetResourcesResponse resources; + protected AbstractGetResourcesResponse() {} + protected AbstractGetResourcesResponse(QueryPage resources) { this.resources = Objects.requireNonNull(resources); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java index 71ebdc73e77ff..264b996b3e8f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsAction.java @@ -9,9 +9,11 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.core.ml.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import java.io.IOException; import java.util.Collections; public class GetDataFrameAnalyticsAction extends Action { @@ -30,6 +32,12 @@ public Response newResponse() { public static class Request extends AbstractGetResourcesRequest { + public Request() {} + + public Request(StreamInput in) throws IOException { + readFrom(in); + } + @Override public String getResourceIdField() { return DataFrameAnalyticsConfig.ID.getPreferredName(); @@ -40,6 +48,8 @@ public static class Response extends AbstractGetResourcesResponse analytics) { super(analytics); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java new file mode 100644 index 0000000000000..e3b7262095abb --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsActionResponseTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction.Response; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; + +import java.util.ArrayList; +import java.util.List; + +public class GetDataFrameAnalyticsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List analytics = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + analytics.add(DataFrameAnalyticsConfigTests.createRandom(DataFrameAnalyticsConfigTests.randomValidId())); + } + return new Response(new QueryPage<>(analytics, analytics.size(), Response.RESULTS_FIELD)); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java new file mode 100644 index 0000000000000..48381526b8c4c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsRequestTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsAction.Request; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; + +public class GetDataFrameAnalyticsRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(); + request.setResourceId(randomAlphaOfLength(20)); + request.setPageParams(new PageParams(randomIntBetween(0, 100), randomIntBetween(0, 100))); + return request; + } + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java new file mode 100644 index 0000000000000..65c51faa9157e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsActionResponseTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Response; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsState; + +import java.util.ArrayList; +import java.util.List; + +public class GetDataFrameAnalyticsStatsActionResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List analytics = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + Response.Stats stats = new Response.Stats(DataFrameAnalyticsConfigTests.randomValidId(), + randomFrom(DataFrameAnalyticsState.values()), null, randomAlphaOfLength(20)); + analytics.add(stats); + } + return new Response(new QueryPage<>(analytics, analytics.size(), GetDataFrameAnalyticsAction.Response.RESULTS_FIELD)); + } + + @Override + protected Writeable.Reader instanceReader() { + return Response::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java new file mode 100644 index 0000000000000..8db7d8db7877c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDataFrameAnalyticsStatsRequestTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ml.action.GetDataFrameAnalyticsStatsAction.Request; +import org.elasticsearch.xpack.core.ml.action.util.PageParams; + +public class GetDataFrameAnalyticsStatsRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAlphaOfLength(20)); + request.setPageParams(new PageParams(randomIntBetween(0, 100), randomIntBetween(0, 100))); + return request; + } + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java new file mode 100644 index 0000000000000..011044fb96eef --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDataFrameAnalyticsActionResponseTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction.Response; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; + +public class PutDataFrameAnalyticsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(DataFrameAnalyticsConfigTests.createRandom(DataFrameAnalyticsConfigTests.randomValidId())); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java new file mode 100644 index 0000000000000..9875c87f5ef3c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsRequestTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.ml.action.StartDataFrameAnalyticsAction.Request; + +public class StartDataFrameAnalyticsRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAlphaOfLength(20)); + } + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } +} diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 43421b4591f0a..2ddcd3f276027 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -35,6 +35,17 @@ integTestRunner { 'ml/datafeeds_crud/Test put datafeed with invalid query', 'ml/datafeeds_crud/Test put datafeed with security headers in the body', 'ml/datafeeds_crud/Test update datafeed with missing id', + 'ml/data_frame_analytics_crud/Test put config with inconsistent body/param ids', + 'ml/data_frame_analytics_crud/Test put config with invalid id', + 'ml/data_frame_analytics_crud/Test put config with unknown top level field', + 'ml/data_frame_analytics_crud/Test put config with unknown field in outlier detection analysis', + 'ml/data_frame_analytics_crud/Test put config given missing source', + 'ml/data_frame_analytics_crud/Test put config given missing dest', + 'ml/data_frame_analytics_crud/Test put config given missing analyses', + 'ml/data_frame_analytics_crud/Test put config given empty analyses', + 'ml/data_frame_analytics_crud/Test put config given two analyses', + 'ml/data_frame_analytics_crud/Test get given missing analytics', + 'ml/data_frame_analytics_crud/Test delete given missing config', 'ml/delete_job_force/Test cannot force delete a non-existent job', 'ml/delete_model_snapshot/Test delete snapshot missing snapshotId', 'ml/delete_model_snapshot/Test delete snapshot missing job_id', @@ -84,6 +95,9 @@ integTestRunner { 'ml/post_data/Test POST data with invalid parameters', 'ml/preview_datafeed/Test preview missing datafeed', 'ml/revert_model_snapshot/Test revert model with invalid snapshotId', + 'ml/start_data_frame_analytics/Test start given missing config', + 'ml/start_data_frame_analytics/Test start given missing source index', + 'ml/start_data_frame_analytics/Test start given source index has no compatible fields', 'ml/start_stop_datafeed/Test start datafeed job, but not open', 'ml/start_stop_datafeed/Test start non existing datafeed', 'ml/start_stop_datafeed/Test stop non existing datafeed', diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java index 3a02932c39c41..5151d0c0c6e8d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/DataFrameAnalysesUtils.java @@ -33,13 +33,10 @@ static DataFrameAnalysis readAnalysis(DataFrameAnalysisConfig config) { Map configMap = config.asMap(); DataFrameAnalysis.Type analysisType = DataFrameAnalysis.Type.fromString(configMap.keySet().iterator().next()); DataFrameAnalysis.Factory factory = factories.get(analysisType); - if (factory == null) { - throw new ElasticsearchParseException("Unknown analysis type [{}]", analysisType); - } Map analysisConfig = castAsMapAndCopy(analysisType, configMap.get(analysisType.toString())); DataFrameAnalysis dataFrameAnalysis = factory.create(analysisConfig); if (analysisConfig.isEmpty() == false) { - throw new ElasticsearchParseException("Data frame analysis [{}] does not support one or more provided parameters: {}", + throw new ElasticsearchParseException("Data frame analysis [{}] does not support one or more provided parameters {}", analysisType, analysisConfig.keySet()); } return dataFrameAnalysis; @@ -62,8 +59,8 @@ static Integer readInt(DataFrameAnalysis.Type analysisType, Map try { return (int) value; } catch (ClassCastException e) { - throw new ElasticsearchParseException("Property [{}] of analysis [{}] should be of type int but was [{}]", - property, analysisType, value.getClass().getName()); + throw new ElasticsearchParseException("Property [{}] of analysis [{}] should be of type [Integer] but was [{}]", + property, analysisType, value.getClass().getSimpleName()); } } @@ -76,8 +73,8 @@ static String readString(DataFrameAnalysis.Type analysisType, Map DataFrameAnalysesUtils.readAnalyses(Collections.singletonList(analysisConfig))); + + assertThat(e.getMessage(), equalTo("Unknown analysis type [unknown_analysis]")); + } + + public void testReadAnalysis_GivenAnalysisIsNotAnObject() { + String analysisJson = "{\"outlier_detection\": 42}"; + DataFrameAnalysisConfig analysisConfig = createAnalysisConfig(analysisJson); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> DataFrameAnalysesUtils.readAnalyses(Collections.singletonList(analysisConfig))); + + assertThat(e.getMessage(), equalTo("[outlier_detection] expected to be a map but was of type [java.lang.Integer]")); + } + + public void testReadAnalysis_GivenUnusedParameters() { + String analysisJson = "{\"outlier_detection\": {\"number_neighbours\":42, \"foo\": 1}}"; + DataFrameAnalysisConfig analysisConfig = createAnalysisConfig(analysisJson); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> DataFrameAnalysesUtils.readAnalyses(Collections.singletonList(analysisConfig))); + + assertThat(e.getMessage(), equalTo("Data frame analysis [outlier_detection] does not support one or more provided " + + "parameters [foo]")); + } + + public void testReadAnalysis_GivenValidOutlierDetection() { + String analysisJson = "{\"outlier_detection\": {\"number_neighbours\":42}}"; + DataFrameAnalysisConfig analysisConfig = createAnalysisConfig(analysisJson); + + List analyses = DataFrameAnalysesUtils.readAnalyses(Collections.singletonList(analysisConfig)); + + assertThat(analyses.size(), equalTo(1)); + assertThat(analyses.get(0), is(instanceOf(OutlierDetection.class))); + OutlierDetection outlierDetection = (OutlierDetection) analyses.get(0); + assertThat(outlierDetection.getParams().size(), equalTo(1)); + assertThat(outlierDetection.getParams().get("number_neighbours"), equalTo(42)); + } + + private static DataFrameAnalysisConfig createAnalysisConfig(String json) { + Map asMap = XContentHelper.convertToMap(new BytesArray(json), true, XContentType.JSON).v2(); + return new DataFrameAnalysisConfig(asMap); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetectionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetectionTests.java new file mode 100644 index 0000000000000..59a838acc8cd8 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/analyses/OutlierDetectionTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.dataframe.analyses; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class OutlierDetectionTests extends ESTestCase { + + public void testCreate_GivenNumberNeighboursNotInt() { + Map config = new HashMap<>(); + config.put(OutlierDetection.NUMBER_NEIGHBOURS, "42"); + + DataFrameAnalysis.Factory factory = new OutlierDetection.Factory(); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config)); + assertThat(e.getMessage(), equalTo("Property [number_neighbours] of analysis [outlier_detection] should be of " + + "type [Integer] but was [String]")); + } + + public void testCreate_GivenMethodNotString() { + Map config = new HashMap<>(); + config.put(OutlierDetection.METHOD, 42); + + DataFrameAnalysis.Factory factory = new OutlierDetection.Factory(); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config)); + assertThat(e.getMessage(), equalTo("Property [method] of analysis [outlier_detection] should be of " + + "type [String] but was [Integer]")); + } + + public void testCreate_GivenEmptyParams() { + DataFrameAnalysis.Factory factory = new OutlierDetection.Factory(); + OutlierDetection outlierDetection = (OutlierDetection) factory.create(Collections.emptyMap()); + assertThat(outlierDetection.getParams().isEmpty(), is(true)); + } + + public void testCreate_GivenFullParams() { + Map config = new HashMap<>(); + config.put(OutlierDetection.NUMBER_NEIGHBOURS, 42); + config.put(OutlierDetection.METHOD, "ldof"); + + DataFrameAnalysis.Factory factory = new OutlierDetection.Factory(); + OutlierDetection outlierDetection = (OutlierDetection) factory.create(config); + + assertThat(outlierDetection.getParams().size(), equalTo(2)); + assertThat(outlierDetection.getParams().get(OutlierDetection.NUMBER_NEIGHBOURS), equalTo(42)); + assertThat(outlierDetection.getParams().get(OutlierDetection.METHOD), equalTo(OutlierDetection.Method.LDOF)); + } +} From fb1fc5c957d0e52ad0b1b4977b2f4bc32fc62c25 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 1 Feb 2019 15:35:04 +0200 Subject: [PATCH 09/12] Fix error message in test --- .../rest-api-spec/test/ml/data_frame_analytics_crud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index ad0160647c8fe..85e07d63cba4a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -90,7 +90,7 @@ "Test put config with unknown field in outlier detection analysis": - do: - catch: /Data frame analysis \[outlier_detection\] does not support one or more provided parameters[:] \[unknown_field\]/ + catch: /Data frame analysis \[outlier_detection\] does not support one or more provided parameters \[unknown_field\]/ ml.put_data_frame_analytics: id: "unknown_field" body: > From 2ac526d4680c23fd040df73774cbbeee89d68ad3 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Fri, 1 Feb 2019 16:38:21 +0200 Subject: [PATCH 10/12] Ignore unknown fields for the task state --- .../xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java index 27347336cb07c..5d9b7ba756190 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsTaskState.java @@ -27,7 +27,7 @@ public class DataFrameAnalyticsTaskState implements PersistentTaskState { private final long allocationId; private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, + new ConstructingObjectParser<>(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME, true, a -> new DataFrameAnalyticsTaskState((DataFrameAnalyticsState) a[0], (long) a[1])); static { From 4328de7e34acfbae5add48930b2fad8053204773 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 4 Feb 2019 11:38:41 +0200 Subject: [PATCH 11/12] Add a TODO --- .../ml/action/TransportStartDataFrameAnalyticsAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index fe80ea9474483..e45b16a6c6786 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -46,6 +46,11 @@ import java.util.Objects; import java.util.function.Predicate; +/** + * Starts the persistent task for running data frame analytics. + * + * TODO Add to the upgrade mode action + */ public class TransportStartDataFrameAnalyticsAction extends TransportMasterNodeAction { From 7ee7dbb9e3831b257f2ddd932782ef399a26e393 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 4 Feb 2019 12:13:54 +0200 Subject: [PATCH 12/12] Remove layer for converting analysis name to c++ after harmonizing them. --- .../analyses/AbstractDataFrameAnalysis.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java index a941c1609a287..90bcc839bb361 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/analyses/AbstractDataFrameAnalysis.java @@ -18,21 +18,11 @@ public abstract class AbstractDataFrameAnalysis implements DataFrameAnalysis { @Override public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(NAME, typeToConfigName(getType())); + builder.field(NAME, getType()); builder.field(PARAMETERS, getParams()); builder.endObject(); return builder; } - // TODO Make the c++ analyses names same as in java to get rid of this - private String typeToConfigName(Type type) { - switch (type) { - case OUTLIER_DETECTION: - return "outliers"; - default: - throw new IllegalStateException("Unexpected type: " + type); - } - } - protected abstract Map getParams(); }