From dfa97a501e1852311b5e4b5933c1a11841f626bf Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 2 Oct 2020 08:29:19 -0400 Subject: [PATCH 1/2] [ML] adding for_export flag for ml plugin GET resource APIs (#63092) This adds the new `for_export` flag to the following APIs: - GET _ml/anomaly_detection/ - GET _ml/datafeeds/ - GET _ml/data_frame/analytics/ The flag is designed for cloning or exporting configuration objects to later be put into the same cluster or a separate cluster. The following fields are not returned in the objects: - any field that is not user settable (e.g. version, create_time) - any field that is a calculated default value (e.g. datafeed chunking_config) - any field that would effectively require changing to be of use (e.g. datafeed job_id) - any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by) closes https://github.com/elastic/elasticsearch/issues/63055 --- .../client/MLRequestConverters.java | 13 +- .../ml/GetDataFrameAnalyticsRequest.java | 24 +- .../client/ml/GetDatafeedRequest.java | 23 +- .../client/ml/GetJobRequest.java | 21 +- .../MlClientDocumentationIT.java | 3 + .../ml/get-data-frame-analytics.asciidoc | 3 + .../high-level/ml/get-datafeed.asciidoc | 3 + docs/java-rest/high-level/ml/get-job.asciidoc | 3 + .../apis/get-datafeed.asciidoc | 4 + .../anomaly-detection/apis/get-job.asciidoc | 4 + .../apis/get-dfanalytics.asciidoc | 4 + .../apis/get-trained-models.asciidoc | 4 +- docs/reference/ml/ml-shared.asciidoc | 6 + .../core/ml/datafeed/DatafeedConfig.java | 92 ++++--- .../dataframe/DataFrameAnalyticsConfig.java | 31 ++- .../core/ml/inference/TrainedModelConfig.java | 2 +- .../xpack/core/ml/job/config/Job.java | 54 ++-- .../xpack/core/ml/utils/ToXContentParams.java | 8 + .../ml/integration/MlBasicMultiNodeIT.java | 259 ++++++++++++++++-- .../chunked/ChunkedDataExtractor.java | 4 +- .../datafeeds/RestGetDatafeedsAction.java | 7 + .../RestGetDataFrameAnalyticsAction.java | 8 + .../inference/RestGetTrainedModelsAction.java | 3 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 7 + .../api/ml.get_data_frame_analytics.json | 6 + .../rest-api-spec/api/ml.get_datafeeds.json | 6 + .../rest-api-spec/api/ml.get_jobs.json | 6 + .../test/ml/data_frame_analytics_crud.yml | 34 +++ .../rest-api-spec/test/ml/datafeeds_crud.yml | 20 ++ .../rest-api-spec/test/ml/jobs_get.yml | 16 ++ 30 files changed, 567 insertions(+), 111 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 6575a4aacb719..872a69b1b95c3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -121,7 +121,10 @@ static Request getJob(GetJobRequest getJobRequest) { RequestConverters.Params params = new RequestConverters.Params(); if (getJobRequest.getAllowNoMatch() != null) { - params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch())); + params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); + } + if (getJobRequest.getForExport() != null) { + params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); } request.addParameters(params.asMap()); return request; @@ -270,6 +273,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } + if (getDatafeedRequest.getForExport() != null) { + params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + } request.addParameters(params.asMap()); return request; } @@ -645,7 +651,10 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { } } if (getRequest.getAllowNoMatch() != null) { - params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch())); + params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); + } + if (getRequest.getForExport() != null) { + params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); } request.addParameters(params.asMap()); return request; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 40698c4b528fa..bddc8b4c3e1bc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -23,7 +23,6 @@ import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import java.util.Arrays; import java.util.List; @@ -32,11 +31,13 @@ public class GetDataFrameAnalyticsRequest implements Validatable { - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String ALLOW_NO_MATCH = "allow_no_match"; + public static final String FOR_EXPORT = "for_export"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; + private Boolean forExport; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -58,6 +59,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + /** * Whether to ignore if a wildcard expression matches no data frame analytics. * @@ -94,11 +111,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(forExport, other.forExport) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams); + return Objects.hash(ids, allowNoMatch, forExport, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 07f77c6a6c33e..9c3fd58f1a2ce 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -42,10 +42,12 @@ public class GetDatafeedRequest extends ActionRequest implements ToXContentObjec public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -101,6 +103,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public ActionRequestValidationException validate() { return null; @@ -108,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoMatch); + return Objects.hash(datafeedIds, forExport, allowNoMatch); } @Override @@ -123,7 +141,8 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoMatch, that.allowNoMatch); + Objects.equals(allowNoMatch, that.allowNoMatch) && + Objects.equals(forExport, that.forExport); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 643bee325e7cb..14a695becee80 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -43,10 +43,12 @@ public class GetJobRequest extends ActionRequest implements ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String FOR_EXPORT = "for_export"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; + private Boolean forExport; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -101,6 +103,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param forExport Boolean value indicating if certain fields should be removed + */ + public void setForExport(boolean forExport) { + this.forExport = forExport; + } + + public Boolean getForExport() { + return forExport; + } + @Override public ActionRequestValidationException validate() { return null; @@ -108,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(jobIds, allowNoMatch); + return Objects.hash(jobIds, forExport, allowNoMatch); } @Override @@ -123,6 +141,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && + Objects.equals(forExport, that.forExport) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 29b8f7f38f1b0..ef88c15a1efd2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -343,6 +343,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-job-request // tag::get-job-execute @@ -839,6 +840,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> + request.setForExport(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2864,6 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> + request.setForExport(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc index 20684c0d0200c..695ef5bb52484 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc @@ -23,6 +23,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}. include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> Constructing a new GET request referencing an existing {dfanalytics-job} +<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {dfanalytics-job} is created are removed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ml/get-datafeed.asciidoc b/docs/java-rest/high-level/ml/get-datafeed.asciidoc index a0ca8f29756d3..c8383dcdaaa75 100644 --- a/docs/java-rest/high-level/ml/get-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/get-datafeed.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] contain wildcards. <2> Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified). +<3> Optional boolean value for requesting the datafeed in a format that can +then be put into another cluster. Certain fields that can only be set when +the datafeed is created are removed. [id="{upid}-{api}-response"] ==== Get datafeeds response diff --git a/docs/java-rest/high-level/ml/get-job.asciidoc b/docs/java-rest/high-level/ml/get-job.asciidoc index 3fde9b98f310a..e42995edcf8d7 100644 --- a/docs/java-rest/high-level/ml/get-job.asciidoc +++ b/docs/java-rest/high-level/ml/get-job.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] wildcards. <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}. (This includes `_all` string or when no jobs have been specified). +<3> Optional boolean value for requesting the job in a format that can +then be put into another cluster. Certain fields that can only be set when +the job is created are removed. [id="{upid}-{api}-response"] ==== Get {anomaly-jobs} response diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 649905c44221b..2341268e2fc5d 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -57,6 +57,10 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 655078ad446c7..09bcc04c27f79 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -50,6 +50,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [[ml-get-job-results]] == {api-response-body-title} diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index 2de63dd23a3fa..3284345769314 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -70,6 +70,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] (Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +`for_export`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] + [role="child_attributes"] [[ml-get-dfanalytics-results]] == {api-response-body-title} diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index d3a8a258a5422..76abe17fbb3ef 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -67,9 +67,7 @@ Specifies whether the included model definition should be returned as a JSON map `for_export`:: (Optional, boolean) -Indicates if certain fields should be removed from the model configuration on -retrieval. This allows the model to be in an acceptable format to be retrieved -and then added to another cluster. Default is false. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index a6d3d35e98d1f..b478a119e6f34 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -673,6 +673,12 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] +tag::for-export[] +Indicates if certain fields should be removed from the configuration on +retrieval. This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. +end::for-export[] + tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real time. The default value is either the bucket span for short bucket spans, or, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 7154768863568..70b65e18a8677 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -50,6 +51,8 @@ import java.util.Random; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * Datafeed configuration options. Describes where to proactively pull input * data from. @@ -66,6 +69,9 @@ public class DatafeedConfig extends AbstractDiffable implements private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; + private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); + private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); private static final Logger logger = LogManager.getLogger(DatafeedConfig.class); @@ -480,17 +486,43 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); - builder.field(Job.ID.getPreferredName(), jobId); - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + // We don't include the job_id in export as we assume the PUT will be referring to a new job as well + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + if (chunkingConfig != null) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } else { // Don't include random defaults or unnecessary defaults in export + if (queryDelay.equals(defaultRandomQueryDelay(jobId)) == false) { + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + } + // Indices options are a pretty advanced feature, better to not include them if they are just the default ones + if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } + // Removing the default chunking config as it is determined by OTHER fields + if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig(aggProvider)) == false) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } } - builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); } builder.field(INDICES.getPreferredName(), indices); - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (aggProvider != null) { builder.field(AGGREGATIONS.getPreferredName(), aggProvider.getAggs()); } @@ -502,26 +534,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); - if (chunkingConfig != null) { - builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); - } - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (delayedDataCheckConfig != null) { builder.field(DELAYED_DATA_CHECK_CONFIG.getPreferredName(), delayedDataCheckConfig); } if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } - builder.startObject(INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - builder.endObject(); return builder; } + private static TimeValue defaultRandomQueryDelay(String jobId) { + Random random = new Random(jobId.hashCode()); + long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()).findFirst().getAsLong(); + return TimeValue.timeValueMillis(delayMillis); + } + + private static ChunkingConfig defaultChunkingConfig(@Nullable AggProvider aggProvider) { + if (aggProvider == null || aggProvider.getParsedAggs() == null) { + return ChunkingConfig.newAuto(); + } else { + long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); + if (histogramIntervalMillis <= 0) { + throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); + } + return ChunkingConfig.newManual(TimeValue.timeValueMillis(DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); + } + } + /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types @@ -615,10 +655,6 @@ private TimeValue defaultFrequencyTarget(TimeValue bucketSpan) { public static class Builder { - public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; - private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); - private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); - private String id; private String jobId; private TimeValue queryDelay; @@ -855,25 +891,13 @@ private static void checkHistogramIntervalIsPositive(AggregationBuilder histogra private void setDefaultChunkingConfig() { if (chunkingConfig == null) { - if (aggProvider == null || aggProvider.getParsedAggs() == null) { - chunkingConfig = ChunkingConfig.newAuto(); - } else { - long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); - if (histogramIntervalMillis <= 0) { - throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); - } - chunkingConfig = ChunkingConfig.newManual(TimeValue.timeValueMillis( - DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); - } + chunkingConfig = defaultChunkingConfig(aggProvider); } } private void setDefaultQueryDelay() { if (queryDelay == null) { - Random random = new Random(jobId.hashCode()); - long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()) - .findFirst().getAsLong(); - queryDelay = TimeValue.timeValueMillis(delayMillis); + queryDelay = defaultRandomQueryDelay(jobId); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index cb9e01143b3b2..2a0dbd3e61713 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -34,6 +34,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -230,34 +231,34 @@ public Integer getMaxNumThreads() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), id); + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + } if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } builder.field(SOURCE.getPreferredName(), source); builder.field(DEST.getPreferredName(), dest); - builder.startObject(ANALYSIS.getPreferredName()); builder.field(analysis.getWriteableName(), analysis, new MapParams(Collections.singletonMap(VERSION.getPreferredName(), version == null ? null : version.toString()))); builder.endObject(); - - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); - } if (analyzedFields != null) { builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields); } builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep()); - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } - if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); - } - if (version != null) { - builder.field(VERSION.getPreferredName(), version); - } builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index 2e206bc9ac9cd..1e8d549afa273 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,6 +47,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -54,7 +55,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable { public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; - public static final String FOR_EXPORT = "for_export"; public static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; public static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; private static final Set RESERVED_METADATA_FIELDS = new HashSet<>(Arrays.asList( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 2d734e45771c3..da77c14585195 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -41,6 +42,8 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; + /** * This class represents a configured and created Job. The creation time is set * to the time the object was constructed and the finished time and last @@ -550,11 +553,35 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - - builder.field(ID.getPreferredName(), jobId); - builder.field(JOB_TYPE.getPreferredName(), jobType); - if (jobVersion != null) { - builder.field(JOB_VERSION.getPreferredName(), jobVersion); + if (params.paramAsBoolean(FOR_EXPORT, false) == false) { + builder.field(ID.getPreferredName(), jobId); + builder.field(JOB_TYPE.getPreferredName(), jobType); + if (jobVersion != null) { + builder.field(JOB_VERSION.getPreferredName(), jobVersion); + } + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + if (finishedTime != null) { + builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, + finishedTime.getTime()); + } + if (modelSnapshotId != null) { + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + if (deleting) { + builder.field(DELETING.getPreferredName(), deleting); + } + if (modelSnapshotMinVersion != null) { + builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + } else { + if (customSettings != null) { + HashMap newCustomSettings = new HashMap<>(customSettings); + newCustomSettings.remove("created_by"); + builder.field(CUSTOM_SETTINGS.getPreferredName(), newCustomSettings); + } } if (groups.isEmpty() == false) { builder.field(GROUPS.getPreferredName(), groups); @@ -562,11 +589,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); - if (finishedTime != null) { - builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, - finishedTime.getTime()); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -592,19 +614,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (customSettings != null) { - builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); - } - if (modelSnapshotId != null) { - builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); - } - if (modelSnapshotMinVersion != null) { - builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); - if (deleting) { - builder.field(DELETING.getPreferredName(), deleting); - } builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index 84a4c26d43931..b755c13ce18f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -18,6 +18,14 @@ public final class ToXContentParams { */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + /** + * Parameter to indicate if this XContent serialization should only include fields that are allowed to be used + * on PUT + * + * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between + */ + public static final String FOR_EXPORT = "for_export"; + /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields * should be included or not diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index dcfd97ff4fc5f..2acb421c6e9d2 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -106,31 +107,7 @@ public void testMiniFarequote() throws Exception { } public void testMiniFarequoteWithDatafeeder() throws Exception { - boolean datesHaveNanoSecondResolution = randomBoolean(); - String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; - String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; - String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; - Request createAirlineDataRequest = new Request("PUT", "/airline-data"); - createAirlineDataRequest.setJsonEntity("{" - + " \"mappings\": {" - + " \"properties\": {" - + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," - + " \"airline\": { \"type\":\"keyword\"}," - + " \"responsetime\": { \"type\":\"float\"}" - + " }" - + " }" - + "}"); - client().performRequest(createAirlineDataRequest); - Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); - airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); - client().performRequest(airlineData1); - Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); - airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); - client().performRequest(airlineData2); - - // Ensure all data is searchable - refreshAllIndices(); - + createAndIndexFarequote(); String jobId = "mini-farequote-with-data-feeder-job"; createFarequoteJob(jobId); String datafeedId = "bar"; @@ -265,6 +242,211 @@ public void testMiniFarequoteReopen() throws Exception { client().performRequest(new Request("DELETE", BASE_PATH + "anomaly_detectors/" + jobId)); } + @SuppressWarnings("unchecked") + public void testExportAndPutJob() throws Exception { + String jobId = "test-export-import-job"; + createFarequoteJob(jobId); + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + + XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); + Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDatafeed() throws Exception { + createAndIndexFarequote(); + String jobId = "test-export-import-datafeed"; + createFarequoteJob(jobId); + String datafeedId = jobId + "-datafeed"; + createDatafeed(datafeedId, jobId); + + Response dfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + + //Delete this so we can PUT another datafeed for the same job + client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); + + Map toPut = new HashMap<>(originalDfBody); + toPut.put("job_id", jobId); + XContentBuilder xContentBuilder = jsonBuilder().map(toPut); + Request request = new Request("PUT", BASE_PATH + "datafeeds/" + datafeedId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedDfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + assertThat(originalDfBody, equalTo(importedDfBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Exception { + createAndIndexFarequote(); + String analyticsId = "outlier-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "outlier analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "outliers-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("outlier_detection"); + { + xContentBuilder.field("compute_feature_influence", false); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { + createAndIndexFarequote(); + String analyticsId = "regression-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "regression analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "regression-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("regression"); + { + xContentBuilder.field("dependent_variable", "responsetime"); + xContentBuilder.field("training_percent", 50); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Classification() throws Exception { + createAndIndexFarequote(); + String analyticsId = "classification-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "classification analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "classification-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("classification"); + { + xContentBuilder.field("dependent_variable", "airline"); + xContentBuilder.field("training_percent", 60); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + private Response createDatafeed(String datafeedId, String jobId) throws Exception { XContentBuilder xContentBuilder = jsonBuilder(); xContentBuilder.startObject(); @@ -322,4 +504,31 @@ private static void assertFlushResponse(Response response, boolean expectedFlush assertThat(asMap.get("flushed"), is(true)); assertThat(asMap.get("last_finalized_bucket_end"), equalTo(expectedLastFinalizedBucketEnd)); } + + private void createAndIndexFarequote() throws Exception { + boolean datesHaveNanoSecondResolution = randomBoolean(); + String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; + String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; + String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; + Request createAirlineDataRequest = new Request("PUT", "/airline-data"); + createAirlineDataRequest.setJsonEntity("{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + "}"); + client().performRequest(createAirlineDataRequest); + Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); + airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); + client().performRequest(airlineData1); + Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); + airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); + client().performRequest(airlineData2); + + // Ensure all data is searchable + refreshAllIndices(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java index 16447f2d97ec9..94b0a7277a882 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -320,11 +320,11 @@ private AggregatedDataSummary(double earliestTime, double latestTime, long histo } /** - * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder} + * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} */ @Override public long estimateChunk() { - return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; + return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index ebd346f9846c4..a21dc049dd79d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -19,8 +19,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -61,4 +63,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 639adbc375a3a..bca20e49ce6c1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -16,11 +16,14 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -51,4 +54,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient request.isAllowNoResources())); return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index 6e618b1a650f9..74ce885914368 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -35,6 +35,7 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -99,7 +100,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, TrainedModelConfig.FOR_EXPORT); + return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index 9e40e169377cb..d431c62072284 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -21,8 +21,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class RestGetJobsAction extends BaseRestHandler { @@ -63,4 +65,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(FOR_EXPORT); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 80b4649453f7f..5bd3cd20b4137 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -43,6 +43,12 @@ "type":"int", "description":"specifies a max number of analytics to get", "default":100 + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on data frame analytics PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index 56f5ea49f8525..b7a3760e56a3a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on datafeed PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index 7a1ebaed08ceb..ae3f0299747ad 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true + }, + "for_export": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on job PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index 5a32dde99ede8..c7439e4774088 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2168,3 +2168,37 @@ setup: { "description": "blah" } + +--- +"Test GET config for export": + + - do: + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}} + } + - do: + ml.get_data_frame_analytics: + id: "simple-outlier-detection" + for_export: true + - match: { data_frame_analytics.0.source.index.0: "index-source" } + - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } + - match: { data_frame_analytics.0.dest.index: "index-dest" } + - match: { data_frame_analytics.0.analysis: { + "outlier_detection":{ + "compute_feature_influence": true, + "outlier_fraction": 0.05, + "standardization_enabled": true + } + }} + - is_false: data_frame_analytics.0.create_time + - is_false: data_frame_analytics.0.version + - is_false: data_frame_analytics.0.id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 3811faacd70c0..4172eb67832ab 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -513,3 +513,23 @@ setup: - match: { datafeeds.0.indices_options.ignore_throttled: false } - match: { datafeeds.0.indices_options.allow_no_indices: false } +--- +"Test get datafeed for export": + - do: + ml.put_datafeed: + datafeed_id: test-for-export + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"] + } + - do: + ml.get_datafeeds: + datafeed_id: test-for-export + for_export: true + - match: { datafeeds.0.indices.0: "index-foo"} + - is_false: datafeeds.0.datafeed_id + - is_false: datafeeds.0.job_id + - is_false: datafeeds.0.create_time + - is_false: datafeeds.0.query_delay + - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index 1ef10bfec1472..b305e73069efc 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -86,3 +86,19 @@ setup: - match: { jobs.0.description: "Job 1"} - match: { jobs.1.job_id: "jobs-get-2"} - match: { jobs.1.description: "Job 2"} +--- +"Test get job for export": + + - do: + ml.get_jobs: + job_id: jobs-get-1 + for_export: true + - match: { jobs.0.description: "Job 1"} + - is_false: job_id + - is_false: job_type + - is_false: job_version + - is_false: create_time + - is_false: finished_time + - is_false: model_snapshot_id + - is_false: model_snapshot_min_version + - is_false: deleting From 2e29f19ecdd2323b45eaa960918503b9c8791b26 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 20 Oct 2020 11:28:29 -0400 Subject: [PATCH 2/2] [ML] adding new flag exclude_generated that removes generated fields in GET config APIs (#63899) When exporting and cloning ml configurations in a cluster it can be frustrating to remove all the fields that were generated by the plugin. Especially as the number of these fields change from version to version. This flag, exclude_generated, allows the GET config APIs to return configurations with these generated fields removed. APIs supporting this flag: - GET _ml/anomaly_detection/ - GET _ml/datafeeds/ - GET _ml/data_frame/analytics/ The following fields are not returned in the objects: - any field that is not user settable (e.g. version, create_time) - any field that is a calculated default value (e.g. datafeed chunking_config) - any field that is automatically set via another Elastic stack process (e.g. anomaly job custom_settings.created_by) relates to #63055 --- .../client/MLRequestConverters.java | 16 ++--- .../ml/GetDataFrameAnalyticsRequest.java | 18 +++--- .../client/ml/GetDatafeedRequest.java | 18 +++--- .../client/ml/GetJobRequest.java | 18 +++--- .../client/ml/GetTrainedModelsRequest.java | 18 +++--- .../MlClientDocumentationIT.java | 8 +-- .../apis/get-datafeed.asciidoc | 8 +-- .../anomaly-detection/apis/get-job.asciidoc | 10 ++-- .../apis/get-dfanalytics.asciidoc | 58 +++++++++---------- .../apis/get-trained-models.asciidoc | 4 +- docs/reference/ml/ml-shared.asciidoc | 4 +- .../core/ml/datafeed/DatafeedConfig.java | 9 ++- .../dataframe/DataFrameAnalyticsConfig.java | 6 +- .../core/ml/inference/TrainedModelConfig.java | 6 +- .../xpack/core/ml/job/config/Job.java | 6 +- .../xpack/core/ml/utils/ToXContentParams.java | 2 +- .../ml/integration/MlBasicMultiNodeIT.java | 31 ++++++---- .../xpack/ml/integration/TrainedModelIT.java | 3 +- .../datafeeds/RestGetDatafeedsAction.java | 4 +- .../RestGetDataFrameAnalyticsAction.java | 4 +- .../inference/RestGetTrainedModelsAction.java | 4 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 4 +- .../api/ml.get_data_frame_analytics.json | 2 +- .../rest-api-spec/api/ml.get_datafeeds.json | 2 +- .../rest-api-spec/api/ml.get_jobs.json | 2 +- .../api/ml.get_trained_models.json | 2 +- .../test/ml/data_frame_analytics_crud.yml | 3 +- .../rest-api-spec/test/ml/datafeeds_crud.yml | 4 +- .../rest-api-spec/test/ml/inference_crud.yml | 6 +- .../rest-api-spec/test/ml/jobs_get.yml | 3 +- 30 files changed, 145 insertions(+), 138 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 872a69b1b95c3..71e9580b48eb5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -123,8 +123,8 @@ static Request getJob(GetJobRequest getJobRequest) { if (getJobRequest.getAllowNoMatch() != null) { params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); } - if (getJobRequest.getForExport() != null) { - params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); + if (getJobRequest.getExcludeGenerated() != null) { + params.putParam(GetJobRequest.EXCLUDE_GENERATED, Boolean.toString(getJobRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -273,8 +273,8 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } - if (getDatafeedRequest.getForExport() != null) { - params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + if (getDatafeedRequest.getExcludeGenerated() != null) { + params.putParam(GetDatafeedRequest.EXCLUDE_GENERATED, Boolean.toString(getDatafeedRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -653,8 +653,8 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); } - if (getRequest.getForExport() != null) { - params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); + if (getRequest.getExcludeGenerated() != null) { + params.putParam(GetDataFrameAnalyticsRequest.EXCLUDE_GENERATED, Boolean.toString(getRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -795,8 +795,8 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) if (getTrainedModelsRequest.getTags() != null) { params.putParam(GetTrainedModelsRequest.TAGS, Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getTags())); } - if (getTrainedModelsRequest.getForExport() != null) { - params.putParam(GetTrainedModelsRequest.FOR_EXPORT, Boolean.toString(getTrainedModelsRequest.getForExport())); + if (getTrainedModelsRequest.getExcludeGenerated() != null) { + params.putParam(GetTrainedModelsRequest.EXCLUDE_GENERATED, Boolean.toString(getTrainedModelsRequest.getExcludeGenerated())); } Request request = new Request(HttpGet.METHOD_NAME, endpoint); request.addParameters(params.asMap()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index bddc8b4c3e1bc..724c0b70d78d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -32,12 +32,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable { public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; - private Boolean forExport; + private Boolean excludeGenerated; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -65,14 +65,14 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -111,12 +111,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, forExport, pageParams); + return Objects.hash(ids, allowNoMatch, excludeGenerated, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 9c3fd58f1a2ce..89c30a8d42f09 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -42,12 +42,12 @@ public class GetDatafeedRequest extends ActionRequest implements ToXContentObjec public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -109,14 +109,14 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override @@ -126,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(datafeedIds, forExport, allowNoMatch); + return Objects.hash(datafeedIds, excludeGenerated, allowNoMatch); } @Override @@ -142,7 +142,7 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && Objects.equals(allowNoMatch, that.allowNoMatch) && - Objects.equals(forExport, that.forExport); + Objects.equals(excludeGenerated, that.excludeGenerated); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 14a695becee80..db73f90c402f0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -43,12 +43,12 @@ public class GetJobRequest extends ActionRequest implements ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -109,14 +109,14 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override @@ -126,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(jobIds, forExport, allowNoMatch); + return Objects.hash(jobIds, excludeGenerated, allowNoMatch); } @Override @@ -141,7 +141,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && - Objects.equals(forExport, that.forExport) && + Objects.equals(excludeGenerated, that.excludeGenerated) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java index 48f9563f78185..f9be3032dfb18 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java @@ -39,7 +39,7 @@ public class GetTrainedModelsRequest implements Validatable { private static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; public static final String TAGS = "tags"; public static final String INCLUDE = "include"; @@ -48,7 +48,7 @@ public class GetTrainedModelsRequest implements Validatable { private Boolean allowNoMatch; private Set includes = new HashSet<>(); private Boolean decompressDefinition; - private Boolean forExport; + private Boolean excludeGenerated; private PageParams pageParams; private List tags; @@ -163,8 +163,8 @@ public GetTrainedModelsRequest setTags(String... tags) { return setTags(Arrays.asList(tags)); } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -173,10 +173,10 @@ public Boolean getForExport() { * This is useful when getting the model and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed from the mode on GET + * @param excludeGenerated Boolean value indicating if certain fields should be removed from the mode on GET */ - public GetTrainedModelsRequest setForExport(Boolean forExport) { - this.forExport = forExport; + public GetTrainedModelsRequest setExcludeGenerated(Boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; return this; } @@ -198,12 +198,12 @@ public boolean equals(Object o) { && Objects.equals(allowNoMatch, other.allowNoMatch) && Objects.equals(decompressDefinition, other.decompressDefinition) && Objects.equals(includes, other.includes) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, forExport); + return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, excludeGenerated); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index ef88c15a1efd2..1f45f93095e75 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -343,7 +343,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-job-request // tag::get-job-execute @@ -840,7 +840,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2866,7 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> - request.setForExport(false); // <2> + request.setExcludeGenerated(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute @@ -3731,7 +3731,7 @@ public void testGetTrainedModels() throws Exception { .setDecompressDefinition(false) // <6> .setAllowNoMatch(true) // <7> .setTags("regression") // <8> - .setForExport(false); // <9> + .setExcludeGenerated(false); // <9> // end::get-trained-models-request request.setTags((List)null); diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 2341268e2fc5d..a1ea57439339a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} @@ -57,9 +57,9 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 09bcc04c27f79..048c76c9d40c5 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -50,9 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-job-results]] == {api-response-body-title} @@ -63,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished. Otherwise, it is `null`. This property is informational; you cannot change its @@ -83,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index 3284345769314..3a70a10ea9aad 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -30,14 +30,14 @@ experimental[] If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -45,12 +45,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -59,20 +59,20 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [role="child_attributes"] [[ml-get-dfanalytics-results]] @@ -80,7 +80,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -91,18 +91,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -114,11 +114,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -130,36 +130,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -179,7 +179,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index 76abe17fbb3ef..bb5b977f28684 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -65,9 +65,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] Specifies whether the included model definition should be returned as a JSON map (`true`) or in a custom compressed format (`false`). Defaults to `true`. -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index b478a119e6f34..24491ff275de2 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -673,11 +673,11 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] -tag::for-export[] +tag::exclude-generated[] Indicates if certain fields should be removed from the configuration on retrieval. This allows the configuration to be in an acceptable format to be retrieved and then added to another cluster. Default is false. -end::for-export[] +end::exclude-generated[] tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 70b65e18a8677..4f016cc36b800 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -51,7 +51,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * Datafeed configuration options. Describes where to proactively pull input @@ -486,10 +486,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); - // We don't include the job_id in export as we assume the PUT will be referring to a new job as well - builder.field(Job.ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), id); + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { builder.field(CONFIG_TYPE.getPreferredName(), TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 2a0dbd3e61713..79b4ee07f6841 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -34,7 +34,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -231,8 +231,8 @@ public Integer getMaxNumThreads() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); + builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (createTime != null) { builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index 1e8d549afa273..22608ca0b6de0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,7 +47,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -315,9 +315,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field(MODEL_ID.getPreferredName(), modelId); // If the model is to be exported for future import to another cluster, these fields are irrelevant. - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(MODEL_ID.getPreferredName(), modelId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(CREATED_BY.getPreferredName(), createdBy); builder.field(VERSION.getPreferredName(), version.toString()); builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index da77c14585195..ab908037727a4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -42,7 +42,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * This class represents a configured and created Job. The creation time is set @@ -553,8 +553,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(JOB_TYPE.getPreferredName(), jobType); if (jobVersion != null) { builder.field(JOB_VERSION.getPreferredName(), jobVersion); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index b755c13ce18f6..f8cfd73eed876 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -24,7 +24,7 @@ public final class ToXContentParams { * * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between */ - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 2acb421c6e9d2..071b731bd9d59 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.yaml.snakeyaml.util.UriEncoder; +import javax.print.attribute.standard.JobStateReason; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -247,8 +248,9 @@ public void testExportAndPutJob() throws Exception { String jobId = "test-export-import-job"; createFarequoteJob(jobId); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + originalJobBody.remove("job_id"); XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); @@ -256,8 +258,9 @@ public void testExportAndPutJob() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + importedJobBody.remove("job_id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -270,8 +273,9 @@ public void testExportAndPutDatafeed() throws Exception { createDatafeed(datafeedId, jobId); Response dfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?exclude_generated=true")); Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + originalDfBody.remove("datafeed_id"); //Delete this so we can PUT another datafeed for the same job client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); @@ -284,8 +288,9 @@ public void testExportAndPutDatafeed() throws Exception { client().performRequest(request); Response importedDfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?exclude_generated=true")); Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + importedDfBody.remove("datafeed_id"); assertThat(originalDfBody, equalTo(importedDfBody)); } @@ -325,8 +330,9 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -334,10 +340,11 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -378,8 +385,9 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -387,10 +395,11 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -431,8 +440,9 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -440,10 +450,11 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java index aa1c9658e76d2..0486ed61de3e3 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java @@ -204,11 +204,12 @@ public void testExportImportModel() throws IOException { getModel = client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "trained_models/" + modelId + - "?include=definition&decompress_definition=false&for_export=true")); + "?include=definition&decompress_definition=false&exclude_generated=true")); assertThat(getModel.getStatusLine().getStatusCode(), equalTo(200)); Map exportedModel = entityAsMap(getModel); Map modelDefinition = ((List>)exportedModel.get("trained_model_configs")).get(0); + modelDefinition.remove("model_id"); String importedModelId = "regression_model_to_import"; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index a21dc049dd79d..095e422d8f93e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -22,7 +22,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -66,6 +66,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index bca20e49ce6c1..8d4f35e2d47bb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -23,7 +23,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -57,6 +57,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index 74ce885914368..1301566c45b07 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -35,7 +35,7 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -100,7 +100,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); + return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, EXCLUDE_GENERATED); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index d431c62072284..13eb167a236ca 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -24,7 +24,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetJobsAction extends BaseRestHandler { @@ -68,6 +68,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 5bd3cd20b4137..04d1995aa4c6f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -44,7 +44,7 @@ "description":"specifies a max number of analytics to get", "default":100 }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index b7a3760e56a3a..90dc0b72b1c72 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index ae3f0299747ad..fce317332c978 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json index 779fe6069cb63..26728c46194a6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json @@ -69,7 +69,7 @@ "type":"list", "description":"A comma-separated list of tags that the model must have." }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index c7439e4774088..cb96f9aa5ca28 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2188,7 +2188,7 @@ setup: - do: ml.get_data_frame_analytics: id: "simple-outlier-detection" - for_export: true + exclude_generated: true - match: { data_frame_analytics.0.source.index.0: "index-source" } - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } - match: { data_frame_analytics.0.dest.index: "index-dest" } @@ -2201,4 +2201,3 @@ setup: }} - is_false: data_frame_analytics.0.create_time - is_false: data_frame_analytics.0.version - - is_false: data_frame_analytics.0.id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 4172eb67832ab..7a25b62ab83ca 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -526,10 +526,8 @@ setup: - do: ml.get_datafeeds: datafeed_id: test-for-export - for_export: true + exclude_generated: true - match: { datafeeds.0.indices.0: "index-foo"} - - is_false: datafeeds.0.datafeed_id - - is_false: datafeeds.0.job_id - is_false: datafeeds.0.create_time - is_false: datafeeds.0.query_delay - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml index 15c698f1d94c2..3b6dc6f5413ba 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml @@ -854,20 +854,20 @@ setup: } } --- -"Test for_export flag": +"Test exclude_generated flag": - do: ml.get_trained_models: model_id: "a-regression-model-1" - for_export: true + exclude_generated: true include: "definition" decompress_definition: false - match: { trained_model_configs.0.description: "empty model for tests" } + - match: { trained_model_configs.0.model_id: "a-regression-model-1" } - is_true: trained_model_configs.0.compressed_definition - is_true: trained_model_configs.0.input - is_true: trained_model_configs.0.inference_config - is_true: trained_model_configs.0.tags - - is_false: trained_model_configs.0.model_id - is_false: trained_model_configs.0.created_by - is_false: trained_model_configs.0.version - is_false: trained_model_configs.0.create_time diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index b305e73069efc..d884ff7aecab2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -92,9 +92,8 @@ setup: - do: ml.get_jobs: job_id: jobs-get-1 - for_export: true + exclude_generated: true - match: { jobs.0.description: "Job 1"} - - is_false: job_id - is_false: job_type - is_false: job_version - is_false: create_time