[Transform] add new exclude_generated flag to GET transform (#63093) (#63947)

This adds a new flag `exclude_generated` for GET transform API.

This flag is useful for when a transform needs to be cloned within a cluster or exported/imported between clusters.

It removes certain fields that are not able to be set via the PUT api (e.g. version, create_time).

relates #63055

Author: benwtrent

client/rest-high-level/src/main/java/org/elasticsearch/client/TransformRequestConverters.java

@@ -40,6 +40,7 @@
 import static org.elasticsearch.client.RequestConverters.createEntity;
 import static org.elasticsearch.client.transform.DeleteTransformRequest.FORCE;
 import static org.elasticsearch.client.transform.GetTransformRequest.ALLOW_NO_MATCH;
+import static org.elasticsearch.client.transform.GetTransformRequest.EXCLUDE_GENERATED;
 import static org.elasticsearch.client.transform.PutTransformRequest.DEFER_VALIDATION;
 import static org.elasticsearch.client.transform.StopTransformRequest.WAIT_FOR_CHECKPOINT;
 
@@ -89,6 +90,9 @@ static Request getTransform(GetTransformRequest getRequest) {
         if (getRequest.getAllowNoMatch() != null) {
             request.addParameter(ALLOW_NO_MATCH, getRequest.getAllowNoMatch().toString());
         }
+        if (getRequest.getExcludeGenerated() != null) {
+            request.addParameter(EXCLUDE_GENERATED, getRequest.getExcludeGenerated().toString());
+        }
         return request;
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/GetTransformRequest.java

@@ -30,6 +30,7 @@
 
 public class GetTransformRequest implements Validatable {
 
+    public static final String EXCLUDE_GENERATED = "exclude_generated";
     public static final String ALLOW_NO_MATCH = "allow_no_match";
     /**
      * Helper method to create a request that will get ALL Transforms
@@ -42,6 +43,7 @@ public static GetTransformRequest getAllTransformRequest() {
     private final List<String> ids;
     private PageParams pageParams;
     private Boolean allowNoMatch;
+    private Boolean excludeGenerated;
 
     public GetTransformRequest(String... ids) {
         this.ids = Arrays.asList(ids);
@@ -67,6 +69,14 @@ public void setAllowNoMatch(Boolean allowNoMatch) {
         this.allowNoMatch = allowNoMatch;
     }
 
+    public void setExcludeGenerated(boolean excludeGenerated) {
+        this.excludeGenerated = excludeGenerated;
+    }
+
+    public Boolean getExcludeGenerated() {
+        return excludeGenerated;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         if (ids == null || ids.isEmpty()) {
@@ -80,7 +90,7 @@ public Optional<ValidationException> validate() {
 
     @Override
     public int hashCode() {
-        return Objects.hash(ids, pageParams, allowNoMatch);
+        return Objects.hash(ids, pageParams, excludeGenerated, allowNoMatch);
     }
 
     @Override
@@ -95,6 +105,7 @@ public boolean equals(Object obj) {
         GetTransformRequest other = (GetTransformRequest) obj;
         return Objects.equals(ids, other.ids)
             && Objects.equals(pageParams, other.pageParams)
+            && Objects.equals(excludeGenerated, other.excludeGenerated)
             && Objects.equals(allowNoMatch, other.allowNoMatch);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java

@@ -687,6 +687,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
             // tag::get-transform-request-options
             request.setPageParams(new PageParams(0, 100)); // <1>
             request.setAllowNoMatch(true); // <2>
+            request.setExcludeGenerated(false); // <3>
             // end::get-transform-request-options
 
             // tag::get-transform-execute

docs/java-rest/high-level/transform/get_transform.asciidoc

@@ -34,6 +34,9 @@ include-tagged::{doc-tests-file}[{api}-request-options]
 {transforms} to skip. `size` specifies the maximum number of
 {transforms} to get. Defaults to `0` and `100` respectively.
 <2> Whether to ignore if a wildcard expression matches no {transforms}.
+<3> Optional boolean value for requesting the {transform} in a format that can
+then be put into another cluster. Certain fields that can only be set when
+the {transform} is created are removed.
 
 
 include::../execution.asciidoc[]

docs/reference/transform/apis/get-transform.asciidoc

@@ -26,12 +26,12 @@ Retrieves configuration information for {transforms}.
 [[get-transform-prereqs]]
 == {api-prereq-title}
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 privileges:
 
-* `monitor_transform` 
+* `monitor_transform`
 
-The built-in `transform_user` role has this privilege. 
+The built-in `transform_user` role has this privilege.
 
 For more information, see <<security-privileges>> and <<built-in-roles>>.
 
@@ -49,7 +49,7 @@ specifying `*` as the `<transform_id>`, or by omitting the `<transform_id>`.
 `<transform_id>`::
 (Optional, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard]
-  
+
 [[get-transform-query-parms]]
 == {api-query-parms-title}
 
@@ -65,6 +65,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms]
 (Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms]
 
+`exclude_generated`::
+(Optional, boolean)
+Excludes fields that were automatically added when creating the transform.
+This allows the configuration to be in an acceptable format to be retrieved
+and then added to another cluster. Default is false.
+
 [[get-transform-response]]
 == {api-response-body-title}
 
@@ -79,13 +85,13 @@ This property is informational; you cannot change its value.
 `version`::
 (string) The version of {es} that existed on the node when the {transform} was
 created.
-  
+
 [[get-transform-response-codes]]
 == {api-response-codes-title}
 
 `404` (Missing resources)::
   If `allow_no_match` is `false`, this code indicates that there are no
-  resources that match the request or only partial matches for the request.  
+  resources that match the request or only partial matches for the request.
 
 [[get-transform-example]]
 == {api-examples-title}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java

@@ -84,6 +84,7 @@ public final class TransformField {
      * API's)
      */
     public static final String FOR_INTERNAL_STORAGE = "for_internal_storage";
+    public static final String EXCLUDE_GENERATED = "exclude_generated";
 
     // internal document id
     public static String DOCUMENT_ID_FIELD = "_id";

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfig.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.license.RemoteClusterLicenseChecker;
+import org.elasticsearch.xpack.core.transform.TransformField;
 import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper;
 
 import java.io.IOException;
@@ -113,7 +114,11 @@ public void writeTo(StreamOutput out) throws IOException {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.array(INDEX.getPreferredName(), index);
-        builder.field(QUERY.getPreferredName(), queryConfig);
+        if (params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false) == false) {
+            builder.field(QUERY.getPreferredName(), queryConfig);
+        } else if(queryConfig.equals(QueryConfig.matchAll()) == false) {
+            builder.field(QUERY.getPreferredName(), queryConfig);
+        }
         builder.endObject();
         return builder;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

@@ -342,9 +342,31 @@ public void writeTo(final StreamOutput out) throws IOException {
 
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false);
+        final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false);
+        assert (forInternalStorage && excludeGenerated) == false:
+            "unsupported behavior, exclude_generated is true and for_internal_storage is true";
         builder.startObject();
         builder.field(TransformField.ID.getPreferredName(), id);
-        builder.field(TransformField.SOURCE.getPreferredName(), source);
+        if (excludeGenerated == false) {
+            if (headers.isEmpty() == false && forInternalStorage) {
+                builder.field(HEADERS.getPreferredName(), headers);
+            }
+            if (transformVersion != null) {
+                builder.field(TransformField.VERSION.getPreferredName(), transformVersion);
+            }
+            if (createTime != null) {
+                builder.timeField(
+                    TransformField.CREATE_TIME.getPreferredName(),
+                    TransformField.CREATE_TIME.getPreferredName() + "_string",
+                    createTime.toEpochMilli()
+                );
+            }
+            if (forInternalStorage) {
+                builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME);
+            }
+        }
+        builder.field(TransformField.SOURCE.getPreferredName(), source, params);
         builder.field(TransformField.DESTINATION.getPreferredName(), dest);
         if (frequency != null) {
             builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep());
@@ -357,26 +379,10 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
-        if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME);
-        }
-        if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) {
-            builder.field(HEADERS.getPreferredName(), headers);
-        }
         if (description != null) {
             builder.field(TransformField.DESCRIPTION.getPreferredName(), description);
         }
         builder.field(TransformField.SETTINGS.getPreferredName(), settings);
-        if (transformVersion != null) {
-            builder.field(TransformField.VERSION.getPreferredName(), transformVersion);
-        }
-        if (createTime != null) {
-            builder.timeField(
-                TransformField.CREATE_TIME.getPreferredName(),
-                TransformField.CREATE_TIME.getPreferredName() + "_string",
-                createTime.toEpochMilli()
-            );
-        }
         builder.endObject();
         return builder;
     }

x-pack/plugin/src/test/resources/rest-api-spec/api/data_frame_transform_deprecated.get_transform.json

@@ -50,6 +50,12 @@
         "type":"boolean",
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)"
+      },
+      "exclude_generated": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits generated fields. Allows transform configurations to be easily copied between clusters and within the same cluster"
       }
     }
   }

x-pack/plugin/src/test/resources/rest-api-spec/api/transform.get_transform.json

@@ -42,6 +42,12 @@
         "type":"boolean",
         "required":false,
         "description":"Whether to ignore if a wildcard expression matches no transforms. (This includes `_all` string or when no transforms have been specified)"
+      },
+      "exclude_generated": {
+        "required": false,
+        "type": "boolean",
+        "default": false,
+        "description": "Omits fields that are illegal to set on transform PUT"
       }
     }
   }