From 7dfd9b8ed2fde58d6a3480b0cb44639146369dff Mon Sep 17 00:00:00 2001
From: Przemyslaw Witek <przemyslaw.witek@elastic.co>
Date: Wed, 24 Jul 2019 14:41:03 +0200
Subject: [PATCH 1/3] Add result_type field to TimingStats and
 DatafeedTimingStats documents

---
 .../xpack/core/ml/action/GetDatafeedsStatsAction.java       | 3 ++-
 .../xpack/core/ml/action/GetJobsStatsAction.java            | 3 ++-
 .../xpack/core/ml/datafeed/DatafeedTimingStats.java         | 6 +++++-
 .../core/ml/job/process/autodetect/state/TimingStats.java   | 4 ++++
 .../xpack/ml/job/persistence/JobResultsPersisterTests.java  | 2 ++
 5 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
index 91c2eee6a27cb..0197334f56e69 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
@@ -198,7 +198,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     builder.field(
                         TIMING_STATS,
                         timingStats,
-                        new MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true")));
+                        new MapParams(Map.of(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true",
+                                             ToXContentParams.FOR_INTERNAL_STORAGE, "false")));
                 }
                 builder.endObject();
                 return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
index 3b3152e9a48a4..a01e4a95daa89 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
@@ -273,7 +273,8 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                     builder.field(
                         TIMING_STATS,
                         timingStats,
-                        new MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true")));
+                        new MapParams(Map.of(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true",
+                                             ToXContentParams.FOR_INTERNAL_STORAGE, "false")));
                 }
                 return builder;
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
index 775dc9931bc86..6fdb02b0267a7 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
@@ -38,7 +39,7 @@ public class DatafeedTimingStats implements ToXContentObject, Writeable {
     private static ConstructingObjectParser<DatafeedTimingStats, Void> createParser() {
         ConstructingObjectParser<DatafeedTimingStats, Void> parser =
             new ConstructingObjectParser<>(
-                "datafeed_timing_stats",
+                TYPE.getPreferredName(),
                 true,
                 args -> {
                     String jobId = (String) args[0];
@@ -128,6 +129,9 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, true)) {
+            builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName());
+        }
         builder.field(JOB_ID.getPreferredName(), jobId);
         builder.field(SEARCH_COUNT.getPreferredName(), searchCount);
         builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
index b526d614df3ab..1b1866862f20f 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.core.ml.job.results.Result;
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
@@ -195,6 +196,9 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, true)) {
+            builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName());
+        }
         builder.field(Job.ID.getPreferredName(), jobId);
         builder.field(BUCKET_COUNT.getPreferredName(), bucketCount);
         if (params.paramAsBoolean(ToXContentParams.INCLUDE_CALCULATED_FIELDS, false)) {
diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
index da69ef3760a6f..94017ef266f2c 100644
--- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
+++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java
@@ -221,6 +221,7 @@ public void testPersistTimingStats() {
             indexRequest.sourceAsMap(),
             equalTo(
                 Map.of(
+                    "result_type", "timing_stats",
                     "job_id", "foo",
                     "bucket_count", 7,
                     "minimum_bucket_processing_time_ms", 1.0,
@@ -259,6 +260,7 @@ public void testPersistDatafeedTimingStats() {
             indexRequest.sourceAsMap(),
             equalTo(
                 Map.of(
+                    "result_type", "datafeed_timing_stats",
                     "job_id", "foo",
                     "search_count", 6,
                     "bucket_count", 66,

From e2cf1c9ce1a57f39a2ffe9498fa1cfa966464e4f Mon Sep 17 00:00:00 2001
From: Przemyslaw Witek <przemyslaw.witek@elastic.co>
Date: Wed, 24 Jul 2019 15:33:05 +0200
Subject: [PATCH 2/3] Remove unused import

---
 .../xpack/core/ml/action/GetDatafeedsStatsAction.java            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
index 0197334f56e69..123155cb21166 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
@@ -27,7 +27,6 @@
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 

From da847e0f2e516046838a16479fe0b445e942f56f Mon Sep 17 00:00:00 2001
From: Przemyslaw Witek <przemyslaw.witek@elastic.co>
Date: Wed, 24 Jul 2019 16:14:01 +0200
Subject: [PATCH 3/3] Use "false" as a default value for
 ToXContentParams.FOR_INTERNAL_STORAGE parameter and only set it to "true"
 while persisting

---
 .../ml/action/GetDatafeedsStatsAction.java    |  4 +--
 .../core/ml/action/GetJobsStatsAction.java    |  3 +-
 .../core/ml/datafeed/DatafeedTimingStats.java |  2 +-
 .../process/autodetect/state/TimingStats.java |  2 +-
 .../job/persistence/JobResultsPersister.java  | 31 +++++++++++++++----
 5 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
index 123155cb21166..91c2eee6a27cb 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedsStatsAction.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 
@@ -197,8 +198,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                     builder.field(
                         TIMING_STATS,
                         timingStats,
-                        new MapParams(Map.of(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true",
-                                             ToXContentParams.FOR_INTERNAL_STORAGE, "false")));
+                        new MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true")));
                 }
                 builder.endObject();
                 return builder;
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
index a01e4a95daa89..3b3152e9a48a4 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/GetJobsStatsAction.java
@@ -273,8 +273,7 @@ public XContentBuilder toUnwrappedXContent(XContentBuilder builder) throws IOExc
                     builder.field(
                         TIMING_STATS,
                         timingStats,
-                        new MapParams(Map.of(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true",
-                                             ToXContentParams.FOR_INTERNAL_STORAGE, "false")));
+                        new MapParams(Collections.singletonMap(ToXContentParams.INCLUDE_CALCULATED_FIELDS, "true")));
                 }
                 return builder;
             }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
index 6fdb02b0267a7..4e2d51b2ebacd 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedTimingStats.java
@@ -129,7 +129,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
-        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, true)) {
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
             builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName());
         }
         builder.field(JOB_ID.getPreferredName(), jobId);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
index 1b1866862f20f..a99260e668685 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/TimingStats.java
@@ -196,7 +196,7 @@ public void writeTo(StreamOutput out) throws IOException {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, true)) {
+        if (params.paramAsBoolean(ToXContentParams.FOR_INTERNAL_STORAGE, false)) {
             builder.field(Result.RESULT_TYPE.getPreferredName(), TYPE.getPreferredName());
         }
         builder.field(Job.ID.getPreferredName(), jobId);
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
index 1d960c5741836..783706259a17b 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
@@ -38,6 +38,7 @@
 import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
 import org.elasticsearch.xpack.core.ml.job.results.Influencer;
 import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
+import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -130,7 +131,11 @@ private void persistBucketInfluencersStandalone(String jobId, List<BucketInfluen
          * @return this
          */
         public Builder persistTimingStats(TimingStats timingStats) {
-            indexResult(TimingStats.documentId(timingStats.getJobId()), timingStats, TimingStats.TYPE.getPreferredName());
+            indexResult(
+                TimingStats.documentId(timingStats.getJobId()),
+                timingStats,
+                new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
+                TimingStats.TYPE.getPreferredName());
             return this;
         }
 
@@ -185,7 +190,11 @@ public Builder persistForecastRequestStats(ForecastRequestStats forecastRequestS
         }
 
         private void indexResult(String id, ToXContent resultDoc, String resultType) {
-            try (XContentBuilder content = toXContentBuilder(resultDoc)) {
+            indexResult(id, resultDoc, ToXContent.EMPTY_PARAMS, resultType);
+        }
+
+        private void indexResult(String id, ToXContent resultDoc, ToXContent.Params params, String resultType) {
+            try (XContentBuilder content = toXContentBuilder(resultDoc, params)) {
                 bulkRequest.add(new IndexRequest(indexName).id(id).source(content));
             } catch (IOException e) {
                 logger.error(new ParameterizedMessage("[{}] Error serialising {}", jobId, resultType), e);
@@ -335,14 +344,18 @@ public void commitStateWrites(String jobId) {
     public IndexResponse persistDatafeedTimingStats(DatafeedTimingStats timingStats, WriteRequest.RefreshPolicy refreshPolicy) {
         String jobId = timingStats.getJobId();
         logger.trace("[{}] Persisting datafeed timing stats", jobId);
-        Persistable persistable = new Persistable(jobId, timingStats, DatafeedTimingStats.documentId(timingStats.getJobId()));
+        Persistable persistable = new Persistable(
+            jobId,
+            timingStats,
+            new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true")),
+            DatafeedTimingStats.documentId(timingStats.getJobId()));
         persistable.setRefreshPolicy(refreshPolicy);
         return persistable.persist(AnomalyDetectorsIndex.resultsWriteAlias(jobId)).actionGet();
     }
 
-    private XContentBuilder toXContentBuilder(ToXContent obj) throws IOException {
+    private static XContentBuilder toXContentBuilder(ToXContent obj, ToXContent.Params params) throws IOException {
         XContentBuilder builder = jsonBuilder();
-        obj.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        obj.toXContent(builder, params);
         return builder;
     }
 
@@ -350,12 +363,18 @@ private class Persistable {
 
         private final String jobId;
         private final ToXContent object;
+        private final ToXContent.Params params;
         private final String id;
         private WriteRequest.RefreshPolicy refreshPolicy;
 
         Persistable(String jobId, ToXContent object, String id) {
+            this(jobId, object, ToXContent.EMPTY_PARAMS, id);
+        }
+
+        Persistable(String jobId, ToXContent object, ToXContent.Params params, String id) {
             this.jobId = jobId;
             this.object = object;
+            this.params = params;
             this.id = id;
             this.refreshPolicy = WriteRequest.RefreshPolicy.NONE;
         }
@@ -373,7 +392,7 @@ ActionFuture<IndexResponse> persist(String indexName) {
         void persist(String indexName, ActionListener<IndexResponse> listener) {
             logCall(indexName);
 
-            try (XContentBuilder content = toXContentBuilder(object)) {
+            try (XContentBuilder content = toXContentBuilder(object, params)) {
                 IndexRequest indexRequest = new IndexRequest(indexName).id(id).source(content).setRefreshPolicy(refreshPolicy);
                 executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, indexRequest, listener, client::index);
             } catch (IOException e) {