From 707d1be54c810db8eb76aff9e3243c52c980e43e Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 19 Oct 2020 13:54:08 -0400 Subject: [PATCH 1/3] [ML] adding new flag exclude_generated that removes generated fields When exporting and cloning ml configurations in a cluster it can be frustrating to remove all the fields that were generated by the plugin. Especially as the number of these fields change from version to version. This flag, remove_generated, allows the GET config APIs to return configurations with these generated fields removed. relates to #63055 --- .../client/MLRequestConverters.java | 12 ++-- .../ml/GetDataFrameAnalyticsRequest.java | 18 +++--- .../client/ml/GetDatafeedRequest.java | 18 +++--- .../client/ml/GetJobRequest.java | 18 +++--- .../MlClientDocumentationIT.java | 6 +- .../apis/get-datafeed.asciidoc | 8 +-- .../anomaly-detection/apis/get-job.asciidoc | 10 ++-- .../apis/get-dfanalytics.asciidoc | 60 +++++++++---------- .../apis/get-trained-models.asciidoc | 4 +- docs/reference/ml/ml-shared.asciidoc | 4 +- .../core/ml/datafeed/DatafeedConfig.java | 9 ++- .../dataframe/DataFrameAnalyticsConfig.java | 6 +- .../core/ml/inference/TrainedModelConfig.java | 2 +- .../xpack/core/ml/job/config/Job.java | 6 +- .../xpack/core/ml/utils/ToXContentParams.java | 2 +- .../ml/integration/MlBasicMultiNodeIT.java | 31 ++++++---- .../datafeeds/RestGetDatafeedsAction.java | 4 +- .../RestGetDataFrameAnalyticsAction.java | 4 +- .../inference/RestGetTrainedModelsAction.java | 2 +- .../xpack/ml/rest/job/RestGetJobsAction.java | 4 +- .../api/ml.get_data_frame_analytics.json | 2 +- .../rest-api-spec/api/ml.get_datafeeds.json | 2 +- .../rest-api-spec/api/ml.get_jobs.json | 2 +- .../test/ml/data_frame_analytics_crud.yml | 3 +- .../rest-api-spec/test/ml/datafeeds_crud.yml | 4 +- .../rest-api-spec/test/ml/jobs_get.yml | 3 +- 26 files changed, 126 insertions(+), 118 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 872a69b1b95c3..95efb42386c88 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -123,8 +123,8 @@ static Request getJob(GetJobRequest getJobRequest) { if (getJobRequest.getAllowNoMatch() != null) { params.putParam(GetJobRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getJobRequest.getAllowNoMatch())); } - if (getJobRequest.getForExport() != null) { - params.putParam(GetJobRequest.FOR_EXPORT, Boolean.toString(getJobRequest.getForExport())); + if (getJobRequest.getExcludeGenerated() != null) { + params.putParam(GetJobRequest.EXCLUDE_GENERATED, Boolean.toString(getJobRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -273,8 +273,8 @@ static Request getDatafeed(GetDatafeedRequest getDatafeedRequest) { params.putParam(GetDatafeedRequest.ALLOW_NO_MATCH.getPreferredName(), Boolean.toString(getDatafeedRequest.getAllowNoMatch())); } - if (getDatafeedRequest.getForExport() != null) { - params.putParam(GetDatafeedRequest.FOR_EXPORT, Boolean.toString(getDatafeedRequest.getForExport())); + if (getDatafeedRequest.getExcludeGenerated() != null) { + params.putParam(GetDatafeedRequest.EXCLUDE_GENERATED, Boolean.toString(getDatafeedRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; @@ -653,8 +653,8 @@ static Request getDataFrameAnalytics(GetDataFrameAnalyticsRequest getRequest) { if (getRequest.getAllowNoMatch() != null) { params.putParam(GetDataFrameAnalyticsRequest.ALLOW_NO_MATCH, Boolean.toString(getRequest.getAllowNoMatch())); } - if (getRequest.getForExport() != null) { - params.putParam(GetDataFrameAnalyticsRequest.FOR_EXPORT, Boolean.toString(getRequest.getForExport())); + if (getRequest.getExcludeGenerated() != null) { + params.putParam(GetDataFrameAnalyticsRequest.EXCLUDE_GENERATED, Boolean.toString(getRequest.getExcludeGenerated())); } request.addParameters(params.asMap()); return request; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java index bddc8b4c3e1bc..724c0b70d78d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDataFrameAnalyticsRequest.java @@ -32,12 +32,12 @@ public class GetDataFrameAnalyticsRequest implements Validatable { public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private final List ids; private Boolean allowNoMatch; private PageParams pageParams; - private Boolean forExport; + private Boolean excludeGenerated; /** * Helper method to create a request that will get ALL Data Frame Analytics @@ -65,14 +65,14 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -111,12 +111,12 @@ public boolean equals(Object o) { GetDataFrameAnalyticsRequest other = (GetDataFrameAnalyticsRequest) o; return Objects.equals(ids, other.ids) && Objects.equals(allowNoMatch, other.allowNoMatch) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, forExport, pageParams); + return Objects.hash(ids, allowNoMatch, excludeGenerated, pageParams); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java index 38e0b86ffc04f..57b55e8e13b88 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetDatafeedRequest.java @@ -41,12 +41,12 @@ public class GetDatafeedRequest implements Validatable, ToXContentObject { public static final ParseField DATAFEED_IDS = new ParseField("datafeed_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_DATAFEEDS = "_all"; private final List datafeedIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -108,19 +108,19 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override public int hashCode() { - return Objects.hash(datafeedIds, forExport, allowNoMatch); + return Objects.hash(datafeedIds, excludeGenerated, allowNoMatch); } @Override @@ -136,7 +136,7 @@ public boolean equals(Object other) { GetDatafeedRequest that = (GetDatafeedRequest) other; return Objects.equals(datafeedIds, that.datafeedIds) && Objects.equals(allowNoMatch, that.allowNoMatch) && - Objects.equals(forExport, that.forExport); + Objects.equals(excludeGenerated, that.excludeGenerated); } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java index 6ad09de62fdab..7c6909678e886 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetJobRequest.java @@ -42,12 +42,12 @@ public class GetJobRequest implements Validatable, ToXContentObject { public static final ParseField JOB_IDS = new ParseField("job_ids"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; private static final String ALL_JOBS = "_all"; private final List jobIds; private Boolean allowNoMatch; - private Boolean forExport; + private Boolean excludeGenerated; @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -108,19 +108,19 @@ public Boolean getAllowNoMatch() { * This is useful when getting the configuration and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed + * @param excludeGenerated Boolean value indicating if certain fields should be removed */ - public void setForExport(boolean forExport) { - this.forExport = forExport; + public void setExcludeGenerated(boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } @Override public int hashCode() { - return Objects.hash(jobIds, forExport, allowNoMatch); + return Objects.hash(jobIds, excludeGenerated, allowNoMatch); } @Override @@ -135,7 +135,7 @@ public boolean equals(Object other) { GetJobRequest that = (GetJobRequest) other; return Objects.equals(jobIds, that.jobIds) && - Objects.equals(forExport, that.forExport) && + Objects.equals(excludeGenerated, that.excludeGenerated) && Objects.equals(allowNoMatch, that.allowNoMatch); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index b9add106b82d7..f97fd7a3d085c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -342,7 +342,7 @@ public void testGetJob() throws Exception { // tag::get-job-request GetJobRequest request = new GetJobRequest("get-machine-learning-job1", "get-machine-learning-job*"); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-job-request // tag::get-job-execute @@ -839,7 +839,7 @@ public void testGetDatafeed() throws Exception { // tag::get-datafeed-request GetDatafeedRequest request = new GetDatafeedRequest(datafeedId); // <1> request.setAllowNoMatch(true); // <2> - request.setForExport(false); // <3> + request.setExcludeGenerated(false); // <3> // end::get-datafeed-request // tag::get-datafeed-execute @@ -2865,7 +2865,7 @@ public void testGetDataFrameAnalytics() throws Exception { { // tag::get-data-frame-analytics-request GetDataFrameAnalyticsRequest request = new GetDataFrameAnalyticsRequest("my-analytics-config"); // <1> - request.setForExport(false); // <2> + request.setExcludeGenerated(false); // <2> // end::get-data-frame-analytics-request // tag::get-data-frame-analytics-execute diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index ed6442c66ec68..545d74374d929 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -19,7 +19,7 @@ Retrieves configuration information for {dfeeds}. `GET _ml/datafeeds/` + -`GET _ml/datafeeds/_all` +`GET _ml/datafeeds/_all` [[ml-get-datafeed-prereqs]] == {api-prereq-title} @@ -36,7 +36,7 @@ comma-separated list of {dfeeds} or a wildcard expression. You can get information for all {dfeeds} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. +IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. [[ml-get-datafeed-path-parms]] == {api-path-parms-title} @@ -57,9 +57,9 @@ all {dfeeds}. (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-datafeeds] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 668c95c9891a2..0b5b865b2dc9c 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -34,7 +34,7 @@ using a group name, a comma-separated list of jobs, or a wildcard expression. You can get information for all {anomaly-jobs} by using `_all`, by specifying `*` as the ``, or by omitting the ``. -IMPORTANT: This API returns a maximum of 10,000 jobs. +IMPORTANT: This API returns a maximum of 10,000 jobs. [[ml-get-job-path-parms]] == {api-path-parms-title} @@ -50,9 +50,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-defaul (Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-jobs] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-job-results]] == {api-response-body-title} @@ -63,7 +63,7 @@ properties, see <>. `create_time`:: (string) The time the job was created. For example, `1491007356077`. This property is informational; you cannot change its value. - + `finished_time`:: (string) If the job closed or failed, this is the time the job finished, otherwise it is `null`. This property is informational; you cannot change its @@ -83,7 +83,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-example]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index a6adac16dbef5..cd3732339b844 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -27,18 +27,18 @@ experimental[] [[ml-get-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following privileges: * cluster: `monitor_ml` - + For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-desc]] == {api-description-title} -You can get information for multiple {dfanalytics-jobs} in a single API request +You can get information for multiple {dfanalytics-jobs} in a single API request by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. @@ -46,12 +46,12 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. == {api-path-parms-title} ``:: -(Optional, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- -You can get information for all {dfanalytics-jobs} by using _all, by specifying -`*` as the ``, or by omitting the +You can get information for all {dfanalytics-jobs} by using _all, by specifying +`*` as the ``, or by omitting the ``. -- @@ -60,20 +60,20 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying == {api-query-parms-title} `allow_no_match`:: -(Optional, boolean) +(Optional, boolean) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match] `from`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: -(Optional, integer) +(Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] -`for_export`:: +`exclude_generated`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [role="child_attributes"] [[ml-get-dfanalytics-results]] @@ -81,7 +81,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] `data_frame_analytics`:: (array) -An array of {dfanalytics-job} resources, which are sorted by the `id` value in +An array of {dfanalytics-job} resources, which are sorted by the `id` value in ascending order. + .Properties of {dfanalytics-job} resources @@ -92,18 +92,18 @@ ascending order. //Begin analyzed_fields `analyzed_fields`::: -(object) Contains `includes` and/or `excludes` patterns that select which fields +(object) Contains `includes` and/or `excludes` patterns that select which fields are included in the analysis. + .Properties of `analyzed_fields` [%collapsible%open] ===== `excludes`::: -(Optional, array) An array of strings that defines the fields that are excluded +(Optional, array) An array of strings that defines the fields that are excluded from the analysis. - + `includes`::: -(Optional, array) An array of strings that defines the fields that are included +(Optional, array) An array of strings that defines the fields that are included in the analysis. ===== //End analyzed_fields @@ -115,11 +115,11 @@ in the analysis. [%collapsible%open] ===== `index`::: -(string) The _destination index_ that stores the results of the +(string) The _destination index_ that stores the results of the {dfanalytics-job}. `results_field`::: -(string) The name of the field that stores the results of the analysis. Defaults +(string) The name of the field that stores the results of the analysis. Defaults to `ml`. ===== //End dest @@ -131,36 +131,36 @@ to `ml`. (string) The `model_memory_limit` that has been set to the {dfanalytics-job}. `source`::: -(object) The configuration of how the analysis data is sourced. It has an +(object) The configuration of how the analysis data is sourced. It has an `index` parameter and optionally a `query` and a `_source`. + .Properties of `source` [%collapsible%open] ===== `index`::: -(array) Index or indices on which to perform the analysis. It can be a single +(array) Index or indices on which to perform the analysis. It can be a single index or index pattern as well as an array of indices or patterns. - + `query`::: -(object) The query that has been specified for the {dfanalytics-job}. The {es} -query domain-specific language (<>). This value corresponds to -the query object in an {es} search POST body. By default, this property has the +(object) The query that has been specified for the {dfanalytics-job}. The {es} +query domain-specific language (<>). This value corresponds to +the query object in an {es} search POST body. By default, this property has the following value: `{"match_all": {}}`. `_source`::: -(object) Contains the specified `includes` and/or `excludes` patterns that -select which fields are present in the destination. Fields that are excluded +(object) Contains the specified `includes` and/or `excludes` patterns that +select which fields are present in the destination. Fields that are excluded cannot be included in the analysis. + .Properties of `_source` [%collapsible%open] ====== `excludes`::: -(array) An array of strings that defines the fields that are excluded from the +(array) An array of strings that defines the fields that are excluded from the destination. - + `includes`::: -(array) An array of strings that defines the fields that are included in the +(array) An array of strings that defines the fields that are included in the destination. ====== //End of _source @@ -180,7 +180,7 @@ destination. [[ml-get-dfanalytics-example]] == {api-examples-title} -The following example gets configuration information for the `loganalytics` +The following example gets configuration information for the `loganalytics` {dfanalytics-job}: [source,console] diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index 9fbfd09e33966..bd83fec03d220 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -67,7 +67,9 @@ Specifies whether the included model definition should be returned as a JSON map `for_export`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=for-export] +Indicates if certain fields should be removed from the model configuration on +retrieval. This allows the model to be in an acceptable format to be retrieved +and then added to another cluster. Default is false. `from`:: (Optional, integer) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 685546d87bb5a..74bdcf04d58d2 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -678,11 +678,11 @@ The number of individual forecasts currently available for the job. A value of `1` or more indicates that forecasts exist. end::forecast-total[] -tag::for-export[] +tag::exclude-generated[] Indicates if certain fields should be removed from the configuration on retrieval. This allows the configuration to be in an acceptable format to be retrieved and then added to another cluster. Default is false. -end::for-export[] +end::exclude-generated[] tag::frequency[] The interval at which scheduled queries are made while the {dfeed} runs in real diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index e711ba2a71e64..b4ee4992148b7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -50,7 +50,7 @@ import java.util.Random; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * Datafeed configuration options. Describes where to proactively pull input @@ -457,10 +457,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); - // We don't include the job_id in export as we assume the PUT will be referring to a new job as well - builder.field(Job.ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), id); + builder.field(Job.ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) { builder.field(CONFIG_TYPE.getPreferredName(), TYPE); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 0468bc62dd675..ad1e39910ffb8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -34,7 +34,7 @@ import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING; import static org.elasticsearch.common.xcontent.ObjectParser.ValueType.VALUE; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class DataFrameAnalyticsConfig implements ToXContentObject, Writeable { @@ -227,8 +227,8 @@ public Integer getMaxNumThreads() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), id); + builder.field(ID.getPreferredName(), id); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { if (createTime != null) { builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index b25a28efad710..fe88f2ac58170 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,11 +47,11 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; public class TrainedModelConfig implements ToXContentObject, Writeable { + public static final String FOR_EXPORT = "for_export"; public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index a3055dde3e9e5..ba36b671cc648 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -42,7 +42,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; /** * This class represents a configured and created Job. The creation time is set @@ -515,8 +515,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { final String humanReadableSuffix = "_string"; - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(ID.getPreferredName(), jobId); + builder.field(ID.getPreferredName(), jobId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(JOB_TYPE.getPreferredName(), jobType); if (jobVersion != null) { builder.field(JOB_VERSION.getPreferredName(), jobVersion); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java index b755c13ce18f6..f8cfd73eed876 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/ToXContentParams.java @@ -24,7 +24,7 @@ public final class ToXContentParams { * * This helps to GET a configuration, copy it, and then PUT it directly without removing or changing any fields in between */ - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; /** * When serialising POJOs to X Content this indicates whether the calculated (i.e. not stored) fields diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java index 2acb421c6e9d2..071b731bd9d59 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlBasicMultiNodeIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.yaml.snakeyaml.util.UriEncoder; +import javax.print.attribute.standard.JobStateReason; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; @@ -247,8 +248,9 @@ public void testExportAndPutJob() throws Exception { String jobId = "test-export-import-job"; createFarequoteJob(jobId); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("jobs")).get(0); + originalJobBody.remove("job_id"); XContentBuilder xContentBuilder = jsonBuilder().map(originalJobBody); Request request = new Request("PUT", BASE_PATH + "anomaly_detectors/" + jobId + "-import"); @@ -256,8 +258,9 @@ public void testExportAndPutJob() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "anomaly_detectors/" + jobId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse).get("jobs")).get(0); + importedJobBody.remove("job_id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -270,8 +273,9 @@ public void testExportAndPutDatafeed() throws Exception { createDatafeed(datafeedId, jobId); Response dfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "?exclude_generated=true")); Map originalDfBody = (Map)((List) entityAsMap(dfResponse).get("datafeeds")).get(0); + originalDfBody.remove("datafeed_id"); //Delete this so we can PUT another datafeed for the same job client().performRequest(new Request("DELETE", BASE_PATH + "datafeeds/" + datafeedId)); @@ -284,8 +288,9 @@ public void testExportAndPutDatafeed() throws Exception { client().performRequest(request); Response importedDfResponse = client().performRequest( - new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "datafeeds/" + datafeedId + "-import" + "?exclude_generated=true")); Map importedDfBody = (Map)((List) entityAsMap(importedDfResponse).get("datafeeds")).get(0); + importedDfBody.remove("datafeed_id"); assertThat(originalDfBody, equalTo(importedDfBody)); } @@ -325,8 +330,9 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -334,10 +340,11 @@ public void testExportAndPutDataFrameAnalytics_OutlierDetection() throws Excepti client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -378,8 +385,9 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -387,10 +395,11 @@ public void testExportAndPutDataFrameAnalytics_Regression() throws Exception { client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } @@ -431,8 +440,9 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response jobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "?exclude_generated=true")); Map originalJobBody = (Map)((List) entityAsMap(jobResponse).get("data_frame_analytics")).get(0); + originalJobBody.remove("id"); XContentBuilder newBuilder = jsonBuilder().map(originalJobBody); request = new Request("PUT", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import"); @@ -440,10 +450,11 @@ public void testExportAndPutDataFrameAnalytics_Classification() throws Exception client().performRequest(request); Response importedJobResponse = client().performRequest( - new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?for_export=true")); + new Request("GET", BASE_PATH + "data_frame/analytics/" + analyticsId + "-import" + "?exclude_generated=true")); Map importedJobBody = (Map)((List) entityAsMap(importedJobResponse) .get("data_frame_analytics")) .get(0); + importedJobBody.remove("id"); assertThat(originalJobBody, equalTo(importedJobBody)); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java index 35ca39a4e05c0..1f6281e7f53a3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -21,7 +21,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDatafeedsAction extends BaseRestHandler { @@ -57,6 +57,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java index 140b00f0a0bdd..2e91bac05e312 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/dataframe/RestGetDataFrameAnalyticsAction.java @@ -21,7 +21,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetDataFrameAnalyticsAction extends BaseRestHandler { @@ -55,6 +55,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index bc563c823fa06..d20c8e4aa3a9b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -34,7 +34,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig.FOR_EXPORT; public class RestGetTrainedModelsAction extends BaseRestHandler { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java index a90df97bddd44..bc4f7ffd2ce0c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -23,7 +23,7 @@ import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; -import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetJobsAction extends BaseRestHandler { @@ -59,6 +59,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Collections.singleton(FOR_EXPORT); + return Collections.singleton(EXCLUDE_GENERATED); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json index 5bd3cd20b4137..04d1995aa4c6f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_data_frame_analytics.json @@ -44,7 +44,7 @@ "description":"specifies a max number of analytics to get", "default":100 }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json index b7a3760e56a3a..90dc0b72b1c72 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_datafeeds.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no datafeeds. (This includes `_all` string or when no datafeeds have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json index ae3f0299747ad..fce317332c978 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_jobs.json @@ -39,7 +39,7 @@ "description":"Whether to ignore if a wildcard expression matches no jobs. (This includes `_all` string or when no jobs have been specified)", "deprecated":true }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index c7439e4774088..cb96f9aa5ca28 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2188,7 +2188,7 @@ setup: - do: ml.get_data_frame_analytics: id: "simple-outlier-detection" - for_export: true + exclude_generated: true - match: { data_frame_analytics.0.source.index.0: "index-source" } - match: { data_frame_analytics.0.source.query: {"match_all" : {} } } - match: { data_frame_analytics.0.dest.index: "index-dest" } @@ -2201,4 +2201,3 @@ setup: }} - is_false: data_frame_analytics.0.create_time - is_false: data_frame_analytics.0.version - - is_false: data_frame_analytics.0.id diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index 9b740c0c9a039..263b7af433696 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -519,10 +519,8 @@ setup: - do: ml.get_datafeeds: datafeed_id: test-for-export - for_export: true + exclude_generated: true - match: { datafeeds.0.indices.0: "index-foo"} - - is_false: datafeeds.0.datafeed_id - - is_false: datafeeds.0.job_id - is_false: datafeeds.0.create_time - is_false: datafeeds.0.query_delay - is_false: datafeeds.0.chunking_config diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml index b305e73069efc..d884ff7aecab2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get.yml @@ -92,9 +92,8 @@ setup: - do: ml.get_jobs: job_id: jobs-get-1 - for_export: true + exclude_generated: true - match: { jobs.0.description: "Job 1"} - - is_false: job_id - is_false: job_type - is_false: job_version - is_false: create_time From f97b6dedcf3b77e7254982c91ee67708acc415f1 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:39:51 -0400 Subject: [PATCH 2/3] changing trained model flag --- .../client/MLRequestConverters.java | 4 ++-- .../client/ml/GetTrainedModelsRequest.java | 18 +++++++++--------- .../documentation/MlClientDocumentationIT.java | 2 +- .../apis/get-trained-models.asciidoc | 6 ++---- .../core/ml/inference/TrainedModelConfig.java | 6 +++--- .../xpack/ml/integration/TrainedModelIT.java | 2 +- .../inference/RestGetTrainedModelsAction.java | 4 ++-- .../api/ml.get_trained_models.json | 2 +- .../rest-api-spec/test/ml/inference_crud.yml | 6 +++--- 9 files changed, 24 insertions(+), 26 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 95efb42386c88..71e9580b48eb5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -795,8 +795,8 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) if (getTrainedModelsRequest.getTags() != null) { params.putParam(GetTrainedModelsRequest.TAGS, Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getTags())); } - if (getTrainedModelsRequest.getForExport() != null) { - params.putParam(GetTrainedModelsRequest.FOR_EXPORT, Boolean.toString(getTrainedModelsRequest.getForExport())); + if (getTrainedModelsRequest.getExcludeGenerated() != null) { + params.putParam(GetTrainedModelsRequest.EXCLUDE_GENERATED, Boolean.toString(getTrainedModelsRequest.getExcludeGenerated())); } Request request = new Request(HttpGet.METHOD_NAME, endpoint); request.addParameters(params.asMap()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java index 48f9563f78185..f9be3032dfb18 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsRequest.java @@ -39,7 +39,7 @@ public class GetTrainedModelsRequest implements Validatable { private static final String TOTAL_FEATURE_IMPORTANCE = "total_feature_importance"; private static final String FEATURE_IMPORTANCE_BASELINE = "feature_importance_baseline"; public static final String ALLOW_NO_MATCH = "allow_no_match"; - public static final String FOR_EXPORT = "for_export"; + public static final String EXCLUDE_GENERATED = "exclude_generated"; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; public static final String TAGS = "tags"; public static final String INCLUDE = "include"; @@ -48,7 +48,7 @@ public class GetTrainedModelsRequest implements Validatable { private Boolean allowNoMatch; private Set includes = new HashSet<>(); private Boolean decompressDefinition; - private Boolean forExport; + private Boolean excludeGenerated; private PageParams pageParams; private List tags; @@ -163,8 +163,8 @@ public GetTrainedModelsRequest setTags(String... tags) { return setTags(Arrays.asList(tags)); } - public Boolean getForExport() { - return forExport; + public Boolean getExcludeGenerated() { + return excludeGenerated; } /** @@ -173,10 +173,10 @@ public Boolean getForExport() { * This is useful when getting the model and wanting to put it in another cluster. * * Default value is false. - * @param forExport Boolean value indicating if certain fields should be removed from the mode on GET + * @param excludeGenerated Boolean value indicating if certain fields should be removed from the mode on GET */ - public GetTrainedModelsRequest setForExport(Boolean forExport) { - this.forExport = forExport; + public GetTrainedModelsRequest setExcludeGenerated(Boolean excludeGenerated) { + this.excludeGenerated = excludeGenerated; return this; } @@ -198,12 +198,12 @@ public boolean equals(Object o) { && Objects.equals(allowNoMatch, other.allowNoMatch) && Objects.equals(decompressDefinition, other.decompressDefinition) && Objects.equals(includes, other.includes) - && Objects.equals(forExport, other.forExport) + && Objects.equals(excludeGenerated, other.excludeGenerated) && Objects.equals(pageParams, other.pageParams); } @Override public int hashCode() { - return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, forExport); + return Objects.hash(ids, allowNoMatch, pageParams, decompressDefinition, includes, excludeGenerated); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index f97fd7a3d085c..234fb5d80aaac 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -3727,7 +3727,7 @@ public void testGetTrainedModels() throws Exception { .setDecompressDefinition(false) // <6> .setAllowNoMatch(true) // <7> .setTags("regression") // <8> - .setForExport(false); // <9> + .setExcludeGenerated(false); // <9> // end::get-trained-models-request request.setTags((List)null); diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index bd83fec03d220..cf11b6aa7fb1d 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -65,11 +65,9 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] Specifies whether the included model definition should be returned as a JSON map (`true`) or in a custom compressed format (`false`). Defaults to `true`. -`for_export`:: +`exclude_generated`:: (Optional, boolean) -Indicates if certain fields should be removed from the model configuration on -retrieval. This allows the model to be in an acceptable format to be retrieved -and then added to another cluster. Default is false. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java index fe88f2ac58170..c6e70ef960ad9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfig.java @@ -47,11 +47,11 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ml.utils.NamedXContentObjectHelper.writeNamedObject; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class TrainedModelConfig implements ToXContentObject, Writeable { - public static final String FOR_EXPORT = "for_export"; public static final String NAME = "trained_model_config"; public static final int CURRENT_DEFINITION_COMPRESSION_VERSION = 1; public static final String DECOMPRESS_DEFINITION = "decompress_definition"; @@ -304,9 +304,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + builder.field(MODEL_ID.getPreferredName(), modelId); // If the model is to be exported for future import to another cluster, these fields are irrelevant. - if (params.paramAsBoolean(FOR_EXPORT, false) == false) { - builder.field(MODEL_ID.getPreferredName(), modelId); + if (params.paramAsBoolean(EXCLUDE_GENERATED, false) == false) { builder.field(CREATED_BY.getPreferredName(), createdBy); builder.field(VERSION.getPreferredName(), version.toString()); builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java index aa1c9658e76d2..dcb65a6db77e5 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java @@ -204,7 +204,7 @@ public void testExportImportModel() throws IOException { getModel = client().performRequest(new Request("GET", MachineLearning.BASE_PATH + "trained_models/" + modelId + - "?include=definition&decompress_definition=false&for_export=true")); + "?include=definition&decompress_definition=false&exclude_generated=true")); assertThat(getModel.getStatusLine().getStatusCode(), equalTo(200)); Map exportedModel = entityAsMap(getModel); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java index d20c8e4aa3a9b..431df5401864e 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestGetTrainedModelsAction.java @@ -34,7 +34,7 @@ import static java.util.Arrays.asList; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction.Request.ALLOW_NO_MATCH; -import static org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig.FOR_EXPORT; +import static org.elasticsearch.xpack.core.ml.utils.ToXContentParams.EXCLUDE_GENERATED; public class RestGetTrainedModelsAction extends BaseRestHandler { @@ -99,7 +99,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient @Override protected Set responseParams() { - return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, FOR_EXPORT); + return Set.of(TrainedModelConfig.DECOMPRESS_DEFINITION, EXCLUDE_GENERATED); } private static class RestToXContentListenerWithDefaultValues extends RestToXContentListener { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json index 779fe6069cb63..26728c46194a6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.get_trained_models.json @@ -69,7 +69,7 @@ "type":"list", "description":"A comma-separated list of tags that the model must have." }, - "for_export": { + "exclude_generated": { "required": false, "type": "boolean", "default": false, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml index 15c698f1d94c2..3b6dc6f5413ba 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml @@ -854,20 +854,20 @@ setup: } } --- -"Test for_export flag": +"Test exclude_generated flag": - do: ml.get_trained_models: model_id: "a-regression-model-1" - for_export: true + exclude_generated: true include: "definition" decompress_definition: false - match: { trained_model_configs.0.description: "empty model for tests" } + - match: { trained_model_configs.0.model_id: "a-regression-model-1" } - is_true: trained_model_configs.0.compressed_definition - is_true: trained_model_configs.0.input - is_true: trained_model_configs.0.inference_config - is_true: trained_model_configs.0.tags - - is_false: trained_model_configs.0.model_id - is_false: trained_model_configs.0.created_by - is_false: trained_model_configs.0.version - is_false: trained_model_configs.0.create_time From 2a39de4403676a0b32e241d1b80799bde6e7c3a3 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 20 Oct 2020 10:30:14 -0400 Subject: [PATCH 3/3] fixing test --- .../org/elasticsearch/xpack/ml/integration/TrainedModelIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java index dcb65a6db77e5..0486ed61de3e3 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/TrainedModelIT.java @@ -209,6 +209,7 @@ public void testExportImportModel() throws IOException { Map exportedModel = entityAsMap(getModel); Map modelDefinition = ((List>)exportedModel.get("trained_model_configs")).get(0); + modelDefinition.remove("model_id"); String importedModelId = "regression_model_to_import"; try (XContentBuilder builder = XContentFactory.jsonBuilder()) {