Create a separate job solely for the purpose of verifying its timing stats in mixed and upgraded cluster. (#43315)

Author: przemekwitek

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml

@@ -39,10 +39,10 @@
 "Test get old cluster job's timing stats":
   - do:
       ml.get_job_stats:
-        job_id: old-cluster-job
-  - match: { jobs.0.job_id: old-cluster-job }
+        job_id: old-cluster-job-with-ts
+  - match: { jobs.0.job_id: old-cluster-job-with-ts }
   - match: { jobs.0.state: "closed" }
-  - match: { jobs.0.timing_stats.job_id: old-cluster-job }
+  - match: { jobs.0.timing_stats.job_id: old-cluster-job-with-ts }
   - match: { jobs.0.timing_stats.bucket_count: 1 }
   - gte:   { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
   - gte:   { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml

@@ -169,6 +169,64 @@
        index: [".ml-state", ".ml-anomalies-shared"]
        wait_for_status: green
 
+---
+"Put job with timing stats checking on the old cluster and post some data":
+
+  - do:
+      ml.put_job:
+        job_id: old-cluster-job-with-ts
+        body:  >
+          {
+            "description":"Cluster upgrade with timing stats checking",
+            "analysis_config" : {
+                "bucket_span": "60s",
+                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
+            },
+            "analysis_limits" : {
+                "model_memory_limit": "50mb"
+            },
+            "data_description" : {
+                "format":"xcontent",
+                "time_field":"time",
+                "time_format":"epoch"
+            }
+          }
+  - match: { job_id: old-cluster-job-with-ts }
+
+  - do:
+      ml.open_job:
+        job_id: old-cluster-job-with-ts
+
+  - do:
+      ml.post_data:
+        job_id: old-cluster-job-with-ts
+        body:
+          - airline: AAL
+            responsetime: 132.2046
+            sourcetype: post-data-job
+            time: 1403481600
+          - airline: JZA
+            responsetime: 990.4628
+            sourcetype: post-data-job
+            time: 1403481700
+  - match: { processed_record_count: 2 }
+
+  - do:
+      ml.close_job:
+        job_id: old-cluster-job-with-ts
+
+  - do:
+      ml.get_buckets:
+        job_id: old-cluster-job-with-ts
+  - match: { count: 1 }
+
+  # Wait for indices to be fully allocated before
+  # killing the node
+  - do:
+      cluster.health:
+        index: [".ml-state", ".ml-anomalies-shared"]
+        wait_for_status: green
+
 ---
 "Put job with empty strings in the configuration":
   - do:

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml

@@ -113,16 +113,26 @@ setup:
 "Test get old cluster job's timing stats":
   - do:
       ml.get_job_stats:
-        job_id: old-cluster-job
-  - match: { jobs.0.job_id: old-cluster-job }
+        job_id: old-cluster-job-with-ts
+  - match: { jobs.0.job_id: old-cluster-job-with-ts }
   - match: { jobs.0.state: "closed" }
-  - match: { jobs.0.timing_stats.job_id: old-cluster-job }
+  - match: { jobs.0.timing_stats.job_id: old-cluster-job-with-ts }
   - match: { jobs.0.timing_stats.bucket_count: 1 }
   - gte:   { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
   - gte:   { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
   - gte:   { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
   - gte:   { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }
 
+  - do:
+      ml.delete_job:
+        job_id: old-cluster-job-with-ts
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ml.get_jobs:
+        job_id: old-cluster-job-with-ts
+
 ---
 "Test job with pre 6.4 rules":