diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 6575a4aacb719..71e9580b48eb5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -121,7 +121,10 @@ static Request getJob(GetJobRequest getJobRequest) { RequestConverters.Params params = new RequestConverters.Params(); if (getJobRequest.getAllowNoMatch() != null) { - params.putParam("allow_no_match", Boolean.toString(getJobRequest.getAllowNoMatch())); + params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); + } + if (getJobRequest.getExcludeGenerated() != null) { + params.putParam(GetJobRequest.EXCLUDE_GENERATED, Boolean.toString(getJobRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -270,6 +273,9 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } + if (getDatafeedRequest.getExcludeGenerated() != null) { + params.putParam(GetDatafeedRequest.EXCLUDE_GENERATED, Boolean.toString(getDatafeedRequest.getExcludeGenerated())); + } request.addParameters(params.asMap()); return request; } @@ -645,7 +651,10 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { } } if (getRequest.getAllowNoMatch() != null) { - params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getRequest.getAllowNoMatch())); + params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); + } + if (getRequest.getExcludeGenerated() != null) { + params.putParam(GetDataFrameAnalyticsRequest.EXCLUDE_GENERATED, Boolean.toString(getRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -786,8 +795,8 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) if (getTrainedModelsRequest.getTags() != null) { params.putParam(GetTrainedModelsRequest.TAGS, Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getTags())); } - if (getTrainedModelsRequest.getForExport() != null) { - params.putParam(GetTrainedModelsRequest.FOR_EXPORT, Boolean.toString(getTrainedModelsRequest.getForExport())); + if (getTrainedModelsRequest.getExcludeGenerated() != null) { + params.putParam(GetTrainedModelsRequest.EXCLUDE_GENERATED, Boolean.toString(getTrainedModelsRequest.getExcludeGenerated())); } Request request = new Request(HttpGet.METHOD_NAME, endpoint); request.addParameters(params.asMap()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index 40698c4b528fa..724c0b70d78d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -23,7 +23,6 @@ import org.elasticsearch.client.ValidationException; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; import java.util.Arrays; import java.util.List; @@ -32,11 +31,13 @@ public class GetDataFrameAnalyticsRequest implements Validatable { - public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String ALLOW_NO_MATCH = "allow_no_match"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; + private Boolean excludeGenerated; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -58,6 +59,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param excludeGenerated Boolean value indicating if certain fields should be removed + */ + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; + } + + public Boolean getExcludeGenerated() { + return excludeGenerated; + } + /** * Whether to ignore if a wildcard expression matches no data frame analytics. * @@ -94,11 +111,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams); + return Objects.hash(ids, allowNoMatch, excludeGenerated, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 07f77c6a6c33e..89c30a8d42f09 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -42,10 +42,12 @@ public class GetDatafeedRequest extends ActionRequest implements ToXContentObjec public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -101,6 +103,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param excludeGenerated Boolean value indicating if certain fields should be removed + */ + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; + } + + public Boolean getExcludeGenerated() { + return excludeGenerated; + } + @Override public ActionRequestValidationException validate() { return null; @@ -108,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(datafeedIds, allowNoMatch); + return Objects.hash(datafeedIds, excludeGenerated, allowNoMatch); } @Override @@ -123,7 +141,8 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && - Objects.equals(allowNoMatch, that.allowNoMatch); + Objects.equals(allowNoMatch, that.allowNoMatch) && + Objects.equals(excludeGenerated, that.excludeGenerated); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 643bee325e7cb..db73f90c402f0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -43,10 +43,12 @@ public class GetJobRequest extends ActionRequest implements ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -101,6 +103,22 @@ public Boolean getAllowNoMatch() { return allowNoMatch; } + /** + * Setting this flag to `true` removes certain fields from the configuration on retrieval. + * + * This is useful when getting the configuration and wanting to put it in another cluster. + * + * Default value is false. + * @param excludeGenerated Boolean value indicating if certain fields should be removed + */ + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; + } + + public Boolean getExcludeGenerated() { + return excludeGenerated; + } + @Override public ActionRequestValidationException validate() { return null; @@ -108,7 +126,7 @@ public ActionRequestValidationException validate() { @Override public int hashCode() { - return Objects.hash(jobIds, allowNoMatch); + return Objects.hash(jobIds, excludeGenerated, allowNoMatch); } @Override @@ -123,6 +141,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && + Objects.equals(excludeGenerated, that.excludeGenerated) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java index 48f9563f78185..f9be3032dfb18 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java @@ -39,7 +39,7 @@ public class GetTrainedModelsRequest implements Validatable { private static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; public static final String TAGS = "tags"; public static final String INCLUDE = "include"; @@ -48,7 +48,7 @@ public class GetTrainedModelsRequest implements Validatable { private Boolean allowNoMatch; private Set includes = new HashSet<>(); private Boolean decompressDefinition; - private Boolean forExport; + private Boolean excludeGenerated; private PageParams pageParams; private List tags; @@ -163,8 +163,8 @@ public GetTrainedModelsRequest setTags(String... tags) { return setTags(Arrays.asList(tags)); } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -173,10 +173,10 @@ public Boolean getForExport() { * This is useful when getting the model and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed from the mode on GET + * @param excludeGenerated Boolean value indicating if certain fields should be removed from the mode on GET */ - public GetTrainedModelsRequest setForExport(Boolean forExport) { - this.forExport = forExport; + public GetTrainedModelsRequest setExcludeGenerated(Boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; return this; } @@ -198,12 +198,12 @@ public boolean equals(Object o) { && Objects.equals(allowNoMatch, other.allowNoMatch) && Objects.equals(decompressDefinition, other.decompressDefinition) && Objects.equals(includes, other.includes) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, forExport); + return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, excludeGenerated); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 29b8f7f38f1b0..1f45f93095e75 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -343,6 +343,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> + request.setExcludeGenerated(false); // <3> // end::get-job-request // tag::get-job-execute @@ -839,6 +840,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> + request.setExcludeGenerated(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2864,6 +2866,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> + request.setExcludeGenerated(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute @@ -3728,7 +3731,7 @@ public void testGetTrainedModels() throws Exception { .setDecompressDefinition(false) // <6> .setAllowNoMatch(true) // <7> .setTags("regression") // <8> - .setForExport(false); // <9> + .setExcludeGenerated(false); // <9> // end::get-trained-models-request request.setTags((List)null); diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc index 20684c0d0200c..695ef5bb52484 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc @@ -23,6 +23,9 @@ IDs, or the special wildcard `_all` to get all {dfanalytics-jobs}. include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> Constructing a new GET request referencing an existing {dfanalytics-job} +<2> Optional boolean value for requesting the {dfanalytics-job} in a format that can +then be put into another cluster. Certain fields that can only be set when +the {dfanalytics-job} is created are removed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ml/get-datafeed.asciidoc b/docs/java-rest/high-level/ml/get-datafeed.asciidoc index a0ca8f29756d3..c8383dcdaaa75 100644 --- a/docs/java-rest/high-level/ml/get-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/get-datafeed.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] contain wildcards. <2> Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified). +<3> Optional boolean value for requesting the datafeed in a format that can +then be put into another cluster. Certain fields that can only be set when +the datafeed is created are removed. [id="{upid}-{api}-response"] ==== Get datafeeds response diff --git a/docs/java-rest/high-level/ml/get-job.asciidoc b/docs/java-rest/high-level/ml/get-job.asciidoc index 3fde9b98f310a..e42995edcf8d7 100644 --- a/docs/java-rest/high-level/ml/get-job.asciidoc +++ b/docs/java-rest/high-level/ml/get-job.asciidoc @@ -25,6 +25,9 @@ include-tagged::{doc-tests-file}[{api}-request] wildcards. <2> Whether to ignore if a wildcard expression matches no {anomaly-jobs}. (This includes `_all` string or when no jobs have been specified). +<3> Optional boolean value for requesting the job in a format that can +then be put into another cluster. Certain fields that can only be set when +the job is created are removed. [id="{upid}-{api}-response"] ==== Get {anomaly-jobs} response diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 649905c44221b..a1ea57439339a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} @@ -57,6 +57,10 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] +`exclude_generated`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] + [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 655078ad446c7..048c76c9d40c5 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -50,6 +50,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] +`exclude_generated`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] + [[ml-get-job-results]] == {api-response-body-title} @@ -59,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished. Otherwise, it is `null`. This property is informational; you cannot change its @@ -79,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index 2de63dd23a3fa..3a70a10ea9aad 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -30,14 +30,14 @@ experimental[] If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -45,12 +45,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -59,24 +59,28 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +`exclude_generated`:: +(Optional, boolean) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] + [role="child_attributes"] [[ml-get-dfanalytics-results]] == {api-response-body-title} `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -87,18 +91,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -110,11 +114,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -126,36 +130,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -175,7 +179,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index d3a8a258a5422..bb5b977f28684 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -65,11 +65,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] Specifies whether the included model definition should be returned as a JSON map (`true`) or in a custom compressed format (`false`). Defaults to `true`. -`for_export`:: +`exclude_generated`:: (Optional, boolean) -Indicates if certain fields should be removed from the model configuration on -retrieval. This allows the model to be in an acceptable format to be retrieved -and then added to another cluster. Default is false. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index a6d3d35e98d1f..24491ff275de2 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -673,6 +673,12 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] +tag::exclude-generated[] +Indicates if certain fields should be removed from the configuration on +retrieval. This allows the configuration to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. +end::exclude-generated[] + tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real time. The default value is either the bucket span for short bucket spans, or, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 7154768863568..4f016cc36b800 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -50,6 +51,8 @@ import java.util.Random; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; + /** * Datafeed configuration options. Describes where to proactively pull input * data from. @@ -66,6 +69,9 @@ public class DatafeedConfig extends AbstractDiffable implements private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; + private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); + private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); private static final Logger logger = LogManager.getLogger(DatafeedConfig.class); @@ -482,15 +488,40 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(ID.getPreferredName(), id); builder.field(Job.ID.getPreferredName(), jobId); - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + if (chunkingConfig != null) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } else { // Don't include random defaults or unnecessary defaults in export + if (queryDelay.equals(defaultRandomQueryDelay(jobId)) == false) { + builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + } + // Indices options are a pretty advanced feature, better to not include them if they are just the default ones + if (indicesOptions.equals(SearchRequest.DEFAULT_INDICES_OPTIONS) == false) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } + // Removing the default chunking config as it is determined by OTHER fields + if (chunkingConfig != null && chunkingConfig.equals(defaultChunkingConfig(aggProvider)) == false) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } } - builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); + builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (frequency != null) { builder.field(FREQUENCY.getPreferredName(), frequency.getStringRep()); } builder.field(INDICES.getPreferredName(), indices); - builder.field(QUERY.getPreferredName(), queryProvider.getQuery()); if (aggProvider != null) { builder.field(AGGREGATIONS.getPreferredName(), aggProvider.getAggs()); } @@ -502,26 +533,34 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); - if (chunkingConfig != null) { - builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); - } - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } if (delayedDataCheckConfig != null) { builder.field(DELAYED_DATA_CHECK_CONFIG.getPreferredName(), delayedDataCheckConfig); } if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } - builder.startObject(INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - builder.endObject(); return builder; } + private static TimeValue defaultRandomQueryDelay(String jobId) { + Random random = new Random(jobId.hashCode()); + long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()).findFirst().getAsLong(); + return TimeValue.timeValueMillis(delayMillis); + } + + private static ChunkingConfig defaultChunkingConfig(@Nullable AggProvider aggProvider) { + if (aggProvider == null || aggProvider.getParsedAggs() == null) { + return ChunkingConfig.newAuto(); + } else { + long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); + if (histogramIntervalMillis <= 0) { + throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); + } + return ChunkingConfig.newManual(TimeValue.timeValueMillis(DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); + } + } + /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types @@ -615,10 +654,6 @@ private TimeValue defaultFrequencyTarget(TimeValue bucketSpan) { public static class Builder { - public static final int DEFAULT_AGGREGATION_CHUNKING_BUCKETS = 1000; - private static final TimeValue MIN_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(1); - private static final TimeValue MAX_DEFAULT_QUERY_DELAY = TimeValue.timeValueMinutes(2); - private String id; private String jobId; private TimeValue queryDelay; @@ -855,25 +890,13 @@ private static void checkHistogramIntervalIsPositive(AggregationBuilder histogra private void setDefaultChunkingConfig() { if (chunkingConfig == null) { - if (aggProvider == null || aggProvider.getParsedAggs() == null) { - chunkingConfig = ChunkingConfig.newAuto(); - } else { - long histogramIntervalMillis = ExtractorUtils.getHistogramIntervalMillis(aggProvider.getParsedAggs()); - if (histogramIntervalMillis <= 0) { - throw ExceptionsHelper.badRequestException(Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO); - } - chunkingConfig = ChunkingConfig.newManual(TimeValue.timeValueMillis( - DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis)); - } + chunkingConfig = defaultChunkingConfig(aggProvider); } } private void setDefaultQueryDelay() { if (queryDelay == null) { - Random random = new Random(jobId.hashCode()); - long delayMillis = random.longs(MIN_DEFAULT_QUERY_DELAY.millis(), MAX_DEFAULT_QUERY_DELAY.millis()) - .findFirst().getAsLong(); - queryDelay = TimeValue.timeValueMillis(delayMillis); + queryDelay = defaultRandomQueryDelay(jobId); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index cb9e01143b3b2..79b4ee07f6841 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -34,6 +34,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -231,33 +232,33 @@ public Integer getMaxNumThreads() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { + if (createTime != null) { + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); + } + if (version != null) { + builder.field(VERSION.getPreferredName(), version); + } + if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { + builder.field(CONFIG_TYPE.getPreferredName(), TYPE); + } + } if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } builder.field(SOURCE.getPreferredName(), source); builder.field(DEST.getPreferredName(), dest); - builder.startObject(ANALYSIS.getPreferredName()); builder.field(analysis.getWriteableName(), analysis, new MapParams(Collections.singletonMap(VERSION.getPreferredName(), version == null ? null : version.toString()))); builder.endObject(); - - if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(CONFIG_TYPE.getPreferredName(), TYPE); - } if (analyzedFields != null) { builder.field(ANALYZED_FIELDS.getPreferredName(), analyzedFields); } builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), getModelMemoryLimit().getStringRep()); - if (headers.isEmpty() == false && params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } - if (createTime != null) { - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); - } - if (version != null) { - builder.field(VERSION.getPreferredName(), version); - } builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index 2e206bc9ac9cd..22608ca0b6de0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,6 +47,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class TrainedModelConfig implements ToXContentObject, Writeable { @@ -54,7 +55,6 @@ public class TrainedModelConfig implements ToXContentObject, Writeable { public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; - public static final String FOR_EXPORT = "for_export"; public static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; public static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; private static final Set RESERVED_METADATA_FIELDS = new HashSet<>(Arrays.asList( @@ -315,9 +315,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field(MODEL_ID.getPreferredName(), modelId); // If the model is to be exported for future import to another cluster, these fields are irrelevant. - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(MODEL_ID.getPreferredName(), modelId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(CREATED_BY.getPreferredName(), createdBy); builder.field(VERSION.getPreferredName(), version.toString()); builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 2d734e45771c3..ab908037727a4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -32,6 +32,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -41,6 +42,8 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; + /** * This class represents a configured and created Job. The creation time is set * to the time the object was constructed and the finished time and last @@ -550,11 +553,35 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - builder.field(ID.getPreferredName(), jobId); - builder.field(JOB_TYPE.getPreferredName(), jobType); - if (jobVersion != null) { - builder.field(JOB_VERSION.getPreferredName(), jobVersion); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { + builder.field(JOB_TYPE.getPreferredName(), jobType); + if (jobVersion != null) { + builder.field(JOB_VERSION.getPreferredName(), jobVersion); + } + builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + if (finishedTime != null) { + builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, + finishedTime.getTime()); + } + if (modelSnapshotId != null) { + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + if (deleting) { + builder.field(DELETING.getPreferredName(), deleting); + } + if (modelSnapshotMinVersion != null) { + builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + } else { + if (customSettings != null) { + HashMap newCustomSettings = new HashMap<>(customSettings); + newCustomSettings.remove("created_by"); + builder.field(CUSTOM_SETTINGS.getPreferredName(), newCustomSettings); + } } if (groups.isEmpty() == false) { builder.field(GROUPS.getPreferredName(), groups); @@ -562,11 +589,6 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (description != null) { builder.field(DESCRIPTION.getPreferredName(), description); } - builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); - if (finishedTime != null) { - builder.timeField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, - finishedTime.getTime()); - } builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); if (analysisLimits != null) { builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); @@ -592,19 +614,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th if (resultsRetentionDays != null) { builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); } - if (customSettings != null) { - builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); - } - if (modelSnapshotId != null) { - builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); - } - if (modelSnapshotMinVersion != null) { - builder.field(MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(), modelSnapshotMinVersion); - } builder.field(RESULTS_INDEX_NAME.getPreferredName(), resultsIndexName); - if (deleting) { - builder.field(DELETING.getPreferredName(), deleting); - } builder.field(ALLOW_LAZY_OPEN.getPreferredName(), allowLazyOpen); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index 84a4c26d43931..f8cfd73eed876 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -18,6 +18,14 @@ public final class ToXContentParams { */ public static final String FOR_INTERNAL_STORAGE = "for_internal_storage"; + /** + * Parameter to indicate if this XContent serialization should only include fields that are allowed to be used + * on PUT + * + * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between + */ + public static final String EXCLUDE_GENERATED = "exclude_generated"; + /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields * should be included or not diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index dcfd97ff4fc5f..071b731bd9d59 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -15,9 +15,11 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.yaml.snakeyaml.util.UriEncoder; +import javax.print.attribute.standard.JobStateReason; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -106,31 +108,7 @@ public void testMiniFarequote() throws Exception { } public void testMiniFarequoteWithDatafeeder() throws Exception { - boolean datesHaveNanoSecondResolution = randomBoolean(); - String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; - String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; - String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; - Request createAirlineDataRequest = new Request("PUT", "/airline-data"); - createAirlineDataRequest.setJsonEntity("{" - + " \"mappings\": {" - + " \"properties\": {" - + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," - + " \"airline\": { \"type\":\"keyword\"}," - + " \"responsetime\": { \"type\":\"float\"}" - + " }" - + " }" - + "}"); - client().performRequest(createAirlineDataRequest); - Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); - airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); - client().performRequest(airlineData1); - Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); - airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); - client().performRequest(airlineData2); - - // Ensure all data is searchable - refreshAllIndices(); - + createAndIndexFarequote(); String jobId = "mini-farequote-with-data-feeder-job"; createFarequoteJob(jobId); String datafeedId = "bar"; @@ -265,6 +243,221 @@ public void testMiniFarequoteReopen() throws Exception { client().performRequest(new Request("DELETE", BASE_PATH + "anomaly_detectors/" + jobId)); } + @SuppressWarnings("unchecked") + public void testExportAndPutJob() throws Exception { + String jobId = "test-export-import-job"; + createFarequoteJob(jobId); + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?exclude_generated=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + originalJobBody.remove("job_id"); + + XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); + Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?exclude_generated=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + importedJobBody.remove("job_id"); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDatafeed() throws Exception { + createAndIndexFarequote(); + String jobId = "test-export-import-datafeed"; + createFarequoteJob(jobId); + String datafeedId = jobId + "-datafeed"; + createDatafeed(datafeedId, jobId); + + Response dfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?exclude_generated=true")); + Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + originalDfBody.remove("datafeed_id"); + + //Delete this so we can PUT another datafeed for the same job + client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); + + Map toPut = new HashMap<>(originalDfBody); + toPut.put("job_id", jobId); + XContentBuilder xContentBuilder = jsonBuilder().map(toPut); + Request request = new Request("PUT", BASE_PATH + "datafeeds/" + datafeedId + "-import"); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response importedDfResponse = client().performRequest( + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?exclude_generated=true")); + Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + importedDfBody.remove("datafeed_id"); + assertThat(originalDfBody, equalTo(importedDfBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Exception { + createAndIndexFarequote(); + String analyticsId = "outlier-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "outlier analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "outliers-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("outlier_detection"); + { + xContentBuilder.field("compute_feature_influence", false); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + importedJobBody.remove("id"); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { + createAndIndexFarequote(); + String analyticsId = "regression-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "regression analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "regression-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("regression"); + { + xContentBuilder.field("dependent_variable", "responsetime"); + xContentBuilder.field("training_percent", 50); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + importedJobBody.remove("id"); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + + @SuppressWarnings("unchecked") + public void testExportAndPutDataFrameAnalytics_Classification() throws Exception { + createAndIndexFarequote(); + String analyticsId = "classification-export-import"; + XContentBuilder xContentBuilder = jsonBuilder(); + xContentBuilder.startObject(); + { + xContentBuilder.field("description", "classification analytics"); + + xContentBuilder.startObject("source"); + { + xContentBuilder.field("index", "airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("dest"); + { + xContentBuilder.field("index", "classification-airline-data"); + } + xContentBuilder.endObject(); + xContentBuilder.startObject("analysis"); + { + xContentBuilder.startObject("classification"); + { + xContentBuilder.field("dependent_variable", "airline"); + xContentBuilder.field("training_percent", 60); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + + Request request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId); + request.setJsonEntity(Strings.toString(xContentBuilder)); + client().performRequest(request); + + Response jobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); + Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); + + XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); + request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); + request.setJsonEntity(Strings.toString(newBuilder)); + client().performRequest(request); + + Response importedJobResponse = client().performRequest( + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); + Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) + .get("data_frame_analytics")) + .get(0); + importedJobBody.remove("id"); + assertThat(originalJobBody, equalTo(importedJobBody)); + } + private Response createDatafeed(String datafeedId, String jobId) throws Exception { XContentBuilder xContentBuilder = jsonBuilder(); xContentBuilder.startObject(); @@ -322,4 +515,31 @@ private static void assertFlushResponse(Response response, boolean expectedFlush assertThat(asMap.get("flushed"), is(true)); assertThat(asMap.get("last_finalized_bucket_end"), equalTo(expectedLastFinalizedBucketEnd)); } + + private void createAndIndexFarequote() throws Exception { + boolean datesHaveNanoSecondResolution = randomBoolean(); + String dateMappingType = datesHaveNanoSecondResolution ? "date_nanos" : "date"; + String dateFormat = datesHaveNanoSecondResolution ? "strict_date_optional_time_nanos" : "strict_date_optional_time"; + String randomNanos = datesHaveNanoSecondResolution ? "," + randomIntBetween(100000000, 999999999) : ""; + Request createAirlineDataRequest = new Request("PUT", "/airline-data"); + createAirlineDataRequest.setJsonEntity("{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"" + dateMappingType + "\", \"format\":\"" + dateFormat + "\"}," + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + "}"); + client().performRequest(createAirlineDataRequest); + Request airlineData1 = new Request("PUT", "/airline-data/_doc/1"); + airlineData1.setJsonEntity("{\"time\":\"2016-06-01T00:00:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":135.22}"); + client().performRequest(airlineData1); + Request airlineData2 = new Request("PUT", "/airline-data/_doc/2"); + airlineData2.setJsonEntity("{\"time\":\"2016-06-01T01:59:00" + randomNanos + "Z\",\"airline\":\"AAA\",\"responsetime\":541.76}"); + client().performRequest(airlineData2); + + // Ensure all data is searchable + refreshAllIndices(); + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java index aa1c9658e76d2..0486ed61de3e3 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java @@ -204,11 +204,12 @@ public void testExportImportModel() throws IOException { getModel = client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "trained_models/" + modelId + - "?include=definition&decompress_definition=false&for_export=true")); + "?include=definition&decompress_definition=false&exclude_generated=true")); assertThat(getModel.getStatusLine().getStatusCode(), equalTo(200)); Map exportedModel = entityAsMap(getModel); Map modelDefinition = ((List>)exportedModel.get("trained_model_configs")).get(0); + modelDefinition.remove("model_id"); String importedModelId = "regression_model_to_import"; try (XContentBuilder builder = XContentFactory.jsonBuilder()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java index 16447f2d97ec9..94b0a7277a882 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -320,11 +320,11 @@ private AggregatedDataSummary(double earliestTime, double latestTime, long histo } /** - * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig.Builder} + * This heuristic is a direct copy of the manual chunking config auto-creation done in {@link DatafeedConfig} */ @Override public long estimateChunk() { - return DatafeedConfig.Builder.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; + return DatafeedConfig.DEFAULT_AGGREGATION_CHUNKING_BUCKETS * histogramIntervalMillis; } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index ebd346f9846c4..095e422d8f93e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -19,8 +19,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -61,4 +63,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_DATAFEEDS, request.allowNoMatch()))); return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 639adbc375a3a..8d4f35e2d47bb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -16,11 +16,14 @@ import org.elasticsearch.xpack.ml.MachineLearning; import java.io.IOException; +import java.util.Collections; import java.util.List; +import java.util.Set; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -51,4 +54,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient request.isAllowNoResources())); return channel -> client.execute(GetDataFrameAnalyticsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index 6e618b1a650f9..1301566c45b07 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -35,6 +35,7 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -99,7 +100,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, TrainedModelConfig.FOR_EXPORT); + return org.elasticsearch.common.collect.Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, EXCLUDE_GENERATED); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index 9e40e169377cb..13eb167a236ca 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -21,8 +21,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetJobsAction extends BaseRestHandler { @@ -63,4 +65,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient restRequest.paramAsBoolean(Request.ALLOW_NO_JOBS, request.allowNoMatch()))); return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return Collections.singleton(EXCLUDE_GENERATED); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 80b4649453f7f..04d1995aa4c6f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -43,6 +43,12 @@ "type":"int", "description":"specifies a max number of analytics to get", "default":100 + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on data frame analytics PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index 56f5ea49f8525..90dc0b72b1c72 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on datafeed PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index 7a1ebaed08ceb..fce317332c978 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -38,6 +38,12 @@ "required":false, "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true + }, + "exclude_generated": { + "required": false, + "type": "boolean", + "default": false, + "description": "Omits fields that are illegal to set on job PUT" } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json index 779fe6069cb63..26728c46194a6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json @@ -69,7 +69,7 @@ "type":"list", "description":"A comma-separated list of tags that the model must have." }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index 5a32dde99ede8..cb96f9aa5ca28 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2168,3 +2168,36 @@ setup: { "description": "blah" } + +--- +"Test GET config for export": + + - do: + ml.put_data_frame_analytics: + id: "simple-outlier-detection" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}} + } + - do: + ml.get_data_frame_analytics: + id: "simple-outlier-detection" + exclude_generated: true + - match: { data_frame_analytics.0.source.index.0: "index-source" } + - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } + - match: { data_frame_analytics.0.dest.index: "index-dest" } + - match: { data_frame_analytics.0.analysis: { + "outlier_detection":{ + "compute_feature_influence": true, + "outlier_fraction": 0.05, + "standardization_enabled": true + } + }} + - is_false: data_frame_analytics.0.create_time + - is_false: data_frame_analytics.0.version diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 3811faacd70c0..7a25b62ab83ca 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -513,3 +513,21 @@ setup: - match: { datafeeds.0.indices_options.ignore_throttled: false } - match: { datafeeds.0.indices_options.allow_no_indices: false } +--- +"Test get datafeed for export": + - do: + ml.put_datafeed: + datafeed_id: test-for-export + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"] + } + - do: + ml.get_datafeeds: + datafeed_id: test-for-export + exclude_generated: true + - match: { datafeeds.0.indices.0: "index-foo"} + - is_false: datafeeds.0.create_time + - is_false: datafeeds.0.query_delay + - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml index 15c698f1d94c2..3b6dc6f5413ba 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml @@ -854,20 +854,20 @@ setup: } } --- -"Test for_export flag": +"Test exclude_generated flag": - do: ml.get_trained_models: model_id: "a-regression-model-1" - for_export: true + exclude_generated: true include: "definition" decompress_definition: false - match: { trained_model_configs.0.description: "empty model for tests" } + - match: { trained_model_configs.0.model_id: "a-regression-model-1" } - is_true: trained_model_configs.0.compressed_definition - is_true: trained_model_configs.0.input - is_true: trained_model_configs.0.inference_config - is_true: trained_model_configs.0.tags - - is_false: trained_model_configs.0.model_id - is_false: trained_model_configs.0.created_by - is_false: trained_model_configs.0.version - is_false: trained_model_configs.0.create_time diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index 1ef10bfec1472..d884ff7aecab2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -86,3 +86,18 @@ setup: - match: { jobs.0.description: "Job 1"} - match: { jobs.1.job_id: "jobs-get-2"} - match: { jobs.1.description: "Job 2"} +--- +"Test get job for export": + + - do: + ml.get_jobs: + job_id: jobs-get-1 + exclude_generated: true + - match: { jobs.0.description: "Job 1"} + - is_false: job_type + - is_false: job_version + - is_false: create_time + - is_false: finished_time + - is_false: model_snapshot_id + - is_false: model_snapshot_min_version + - is_false: deleting