From 60f43cc72a6deeebdd66fb4badfc127460549a19 Mon Sep 17 00:00:00 2001
From: Christos Soulios <1561376+csoulios@users.noreply.github.com>
Date: Wed, 28 Apr 2021 17:54:43 +0300
Subject: [PATCH 1/3] Add REST tests for histogram aggregation (#72322)

Related to #26220
---
 .../test/search.aggregation/10_histogram.yml  | 456 ++++++++++++++----
 1 file changed, 351 insertions(+), 105 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index b1b64d86dac6e..a45cd7051615f 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -18,125 +18,418 @@ setup:
 ---
 "Basic test":
   - do:
-      index:
+      bulk:
         index: test_1
-        id: 1
-        body: { "number" : 1 }
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 1}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 151}'
 
   - do:
-      index:
-        index: test_1
-        id: 2
-        body: { "number" : 51 }
+      search:
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 50
 
-  - do:
-      index:
-        index: test_1
-        id: 3
-        body: { "number" : 101 }
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 4 }
+
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - is_false: aggregations.histo.buckets.0.key_as_string
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.1.key: 50 }
+  - is_false: aggregations.histo.buckets.1.key_as_string
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.2.key: 100 }
+  - is_false: aggregations.histo.buckets.2.key_as_string
+  - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
+  - match: { aggregations.histo.buckets.3.key: 150 }
+  - is_false: aggregations.histo.buckets.3.key_as_string
+  - match: { aggregations.histo.buckets.3.doc_count: 1 }
+
+---
+"Format test":
   - do:
-      index:
+      bulk:
         index: test_1
-        id: 4
-        body: { "number" : 151 }
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 1}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 151}'
 
   - do:
-      indices.refresh: {}
+      search:
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 50
+                format: "Value is ##0.0"
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 4 }
 
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "Value is 0.0" }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.1.key: 50 }
+  - match: { aggregations.histo.buckets.1.key_as_string: "Value is 50.0" }
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.2.key: 100 }
+  - match: { aggregations.histo.buckets.2.key_as_string: "Value is 100.0" }
+  - match: { aggregations.histo.buckets.2.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.3.key: 150 }
+  - match: { aggregations.histo.buckets.3.key_as_string: "Value is 150.0" }
+  - match: { aggregations.histo.buckets.3.doc_count: 1 }
+
+---
+"Show empty intervals":
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 1}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 61}'
+          - '{"index": {}}'
+          - '{"number": 71}'
   - do:
       search:
-        rest_total_hits_as_int: true
-        body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50 } } } }
-
-  - match: { hits.total: 4 }
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 20
 
+  - match: { hits.total.value: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
-
   - is_false: aggregations.histo.buckets.0.key_as_string
-
   - match: { aggregations.histo.buckets.0.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.1.key: 50 }
-
+  - match: { aggregations.histo.buckets.1.key: 20 }
   - is_false: aggregations.histo.buckets.1.key_as_string
+  - match: { aggregations.histo.buckets.1.doc_count: 0 }
 
-  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+  - match: { aggregations.histo.buckets.2.key: 40 }
+  - is_false: aggregations.histo.buckets.2.key_as_string
+  - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.2.key: 100 }
+  - match: { aggregations.histo.buckets.3.key: 60 }
+  - is_false: aggregations.histo.buckets.3.key_as_string
+  - match: { aggregations.histo.buckets.3.doc_count: 2 }
+
+---
+"histogram with min_doc_count":
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 1}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 61}'
+          - '{"index": {}}'
+          - '{"number": 71}'
+
+  - do:
+      search: # Test with min_doc_count: 1
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 20
+                min_doc_count: 1
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 3 }
+
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - is_false: aggregations.histo.buckets.0.key_as_string
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
 
+  - match: { aggregations.histo.buckets.1.key: 40 }
+  - is_false: aggregations.histo.buckets.1.key_as_string
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.2.key: 60 }
   - is_false: aggregations.histo.buckets.2.key_as_string
+  - match: { aggregations.histo.buckets.2.doc_count: 2 }
 
+  - do:
+      search: # Test with min_doc_count: 2
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 20
+                min_doc_count: 2
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 1 }
+  - match: { aggregations.histo.buckets.0.key: 60 }
+  - is_false: aggregations.histo.buckets.0.key_as_string
+  - match: { aggregations.histo.buckets.0.doc_count: 2 }
+
+---
+"histogram with extended_bounds":
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 52}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 150}'
+
+  - do:
+      search:
+        body:
+          aggs:
+            histo:
+              histogram:
+                field : number
+                interval : 50
+                extended_bounds:
+                  min: 0
+                  max: 200
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 5 }
+
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - is_false: aggregations.histo.buckets.0.key_as_string
+  - match: { aggregations.histo.buckets.0.doc_count: 0 }
+
+  - match: { aggregations.histo.buckets.1.key: 50 }
+  - is_false: aggregations.histo.buckets.1.key_as_string
+  - match: { aggregations.histo.buckets.1.doc_count: 2 }
+
+  - match: { aggregations.histo.buckets.2.key: 100 }
+  - is_false: aggregations.histo.buckets.2.key_as_string
   - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
   - match: { aggregations.histo.buckets.3.key: 150 }
-
   - is_false: aggregations.histo.buckets.3.key_as_string
-
   - match: { aggregations.histo.buckets.3.doc_count: 1 }
 
+  - match: { aggregations.histo.buckets.4.key: 200 }
+  - is_false: aggregations.histo.buckets.4.key_as_string
+  - match: { aggregations.histo.buckets.4.doc_count: 0 }
+
 ---
-"Format test":
+"histogram with hard bounds":
+  - skip:
+      version: " - 7.9.99"
+      reason:  hard_bounds were introduced in 7.10.0
+
   - do:
-      index:
-        index: test_1
-        id: 1
-        body: { "number" : 1 }
+      indices.create:
+        index: test_3
+        body:
+          mappings:
+            properties:
+              range:
+                type: long_range
 
   - do:
-      index:
-        index: test_1
-        id: 2
-        body: { "number" : 51 }
+      bulk:
+        index: test_3
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"range": {"lte": 10}}'
+          - '{"index": {}}'
+          - '{"range": {"gte": 15}}'
 
   - do:
-      index:
-        index: test_1
-        id: 3
-        body: { "number" : 101 }
+      search:
+        index: test_3
+        body:
+          size: 0
+          aggs:
+            histo:
+              histogram:
+                field: range
+                interval: 1
+                hard_bounds:
+                  min: 0
+                  max: 20
+  - match: { hits.total.value: 2 }
+  - length: { aggregations.histo.buckets: 21 }
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+  - match: { aggregations.histo.buckets.20.key: 20 }
+  - match: { aggregations.histo.buckets.20.doc_count: 1 }
 
+---
+"histogram with offset and unmapped":
   - do:
-      index:
+      bulk:
         index: test_1
-        id: 4
-        body: { "number" : 151 }
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"number": 11}'
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 151}'
+
+  - do:
+      indices.create:
+        index: test_unmapped
 
   - do:
       indices.refresh: {}
 
   - do:
       search:
-        rest_total_hits_as_int: true
-        body: { "aggs" : { "histo" : { "histogram" : { "field" : "number", "interval" : 50, "format" : "Value is ##0.0" } } } }
-
-  - match: { hits.total: 4 }
+        index: test_*
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 50
+                offset: 20
 
+  - match: { hits.total.value: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
-  - match: { aggregations.histo.buckets.0.key: 0 }
+  - match: { aggregations.histo.buckets.0.key: -30 }
+  - is_false: aggregations.histo.buckets.0.key_as_string
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.0.key_as_string: "Value is 0.0" }
+  - match: { aggregations.histo.buckets.1.key: 20 }
+  - is_false: aggregations.histo.buckets.1.key_as_string
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+  - match: { aggregations.histo.buckets.2.key: 70 }
+  - is_false: aggregations.histo.buckets.2.key_as_string
+  - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.1.key: 50 }
+  - match: { aggregations.histo.buckets.3.key: 120 }
+  - is_false: aggregations.histo.buckets.3.key_as_string
+  - match: { aggregations.histo.buckets.3.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.1.key_as_string: "Value is 50.0" }
+---
+"histogram with missing field without missing param":
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"another_number": 11}' # number field is missing
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 151}'
 
-  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+  - do:
+      search:
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 50
 
-  - match: { aggregations.histo.buckets.2.key: 100 }
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 3 }
 
-  - match: { aggregations.histo.buckets.2.key_as_string: "Value is 100.0" }
+  - match: { aggregations.histo.buckets.0.key: 50 }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.1.key: 100 }
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
 
+  - match: { aggregations.histo.buckets.2.key: 150 }
   - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
-  - match: { aggregations.histo.buckets.3.key: 150 }
 
-  - match: { aggregations.histo.buckets.3.key_as_string: "Value is 150.0" }
+---
+"histogram with missing field with missing param":
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"another_number": 11}' # number field is missing
+          - '{"index": {}}'
+          - '{"number": 51}'
+          - '{"index": {}}'
+          - '{"number": 101}'
+          - '{"index": {}}'
+          - '{"number": 151}'
+
+  - do:
+      search:
+        body:
+          aggs:
+            histo:
+              histogram:
+                field: number
+                interval: 50
+                missing: 0
 
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 4 }
+
+  - match: { aggregations.histo.buckets.0.key: 0 }
+  - match: { aggregations.histo.buckets.0.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.1.key: 50 }
+  - match: { aggregations.histo.buckets.1.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.2.key: 100 }
+  - match: { aggregations.histo.buckets.2.doc_count: 1 }
+
+  - match: { aggregations.histo.buckets.3.key: 150 }
   - match: { aggregations.histo.buckets.3.doc_count: 1 }
 
 ---
@@ -325,7 +618,6 @@ setup:
   - match: { aggregations.histo.buckets.2.key_as_string: "2016-02-02T00:00:00.000Z" }
   - match: { aggregations.histo.buckets.2.doc_count: 1 }
 
-
 ---
 "date_histogram on range":
   - skip:
@@ -601,52 +893,6 @@ setup:
   - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 }
   - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
 
----
-"histogram with hard bounds":
-  - skip:
-      version: " - 7.9.99"
-      reason:  hard_bounds were introduced in 7.10.0
-
-  - do:
-      indices.create:
-        index: test_3
-        body:
-          mappings:
-            properties:
-              range:
-                type: long_range
-
-  - do:
-      bulk:
-        index: test_3
-        refresh: true
-        body:
-            - '{"index": {}}'
-            - '{"range": {"lte": 10}}'
-            - '{"index": {}}'
-            - '{"range": {"gte": 15}}'
-
-  - do:
-      search:
-        index: test_3
-        body:
-          size: 0
-          aggs:
-            histo:
-              histogram:
-                field: range
-                interval: 1
-                hard_bounds:
-                  min: 0
-                  max: 20
-  - match: { hits.total.value: 2 }
-  - length: { aggregations.histo.buckets: 21 }
-  - match: { aggregations.histo.buckets.0.key: 0 }
-  - match: { aggregations.histo.buckets.0.doc_count: 1 }
-  - match: { aggregations.histo.buckets.20.key: 20 }
-  - match: { aggregations.histo.buckets.20.doc_count: 1 }
-
-
 ---
 "date_histogram with extended_bounds and offset and unmapped":
   - skip:

From f961a97841a33d8f4e3ae6bc2d086327820b992f Mon Sep 17 00:00:00 2001
From: Christos Soulios <csoulios@gmail.com>
Date: Wed, 28 Apr 2021 23:08:41 +0300
Subject: [PATCH 2/3] Fix broken tests

---
 .../test/search.aggregation/10_histogram.yml   | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index a45cd7051615f..076a796c9f21e 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -33,6 +33,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -40,7 +41,7 @@ setup:
                 field: number
                 interval: 50
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
@@ -77,6 +78,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -85,7 +87,7 @@ setup:
                 interval: 50
                 format: "Value is ##0.0"
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
@@ -121,6 +123,7 @@ setup:
           - '{"number": 71}'
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -128,7 +131,7 @@ setup:
                 field: number
                 interval: 20
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
@@ -325,6 +328,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         index: test_*
         body:
           aggs:
@@ -334,7 +338,7 @@ setup:
                 interval: 50
                 offset: 20
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: -30 }
@@ -371,6 +375,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -378,7 +383,7 @@ setup:
                 field: number
                 interval: 50
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 3 }
 
   - match: { aggregations.histo.buckets.0.key: 50 }
@@ -409,6 +414,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -417,7 +423,7 @@ setup:
                 interval: 50
                 missing: 0
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 4 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }

From 7e6af2b7e1d95a5915eb4dad64217211462c54ad Mon Sep 17 00:00:00 2001
From: Christos Soulios <csoulios@gmail.com>
Date: Wed, 28 Apr 2021 23:22:35 +0300
Subject: [PATCH 3/3] Fix broken tests

---
 .../test/search.aggregation/10_histogram.yml     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index 076a796c9f21e..2a847d618a4cf 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -168,6 +168,7 @@ setup:
 
   - do:
       search: # Test with min_doc_count: 1
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -176,7 +177,7 @@ setup:
                 interval: 20
                 min_doc_count: 1
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 3 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
@@ -193,6 +194,7 @@ setup:
 
   - do:
       search: # Test with min_doc_count: 2
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -201,7 +203,7 @@ setup:
                 interval: 20
                 min_doc_count: 2
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 1 }
   - match: { aggregations.histo.buckets.0.key: 60 }
   - is_false: aggregations.histo.buckets.0.key_as_string
@@ -225,6 +227,7 @@ setup:
 
   - do:
       search:
+        rest_total_hits_as_int: true
         body:
           aggs:
             histo:
@@ -235,7 +238,7 @@ setup:
                   min: 0
                   max: 200
 
-  - match: { hits.total.value: 4 }
+  - match: { hits.total: 4 }
   - length: { aggregations.histo.buckets: 5 }
 
   - match: { aggregations.histo.buckets.0.key: 0 }
@@ -571,11 +574,8 @@ setup:
 ---
 "date_histogram with offset":
   - skip:
-      version: "all"
-      reason:  "AwaitsFix https://github.com/elastic/elasticsearch/issues/51525"
-      # When fixed, reinstate these lines
-      #version: " - 7.1.99"
-      #reason:  calendar_interval introduced in 7.2.0
+      version: " - 7.1.99"
+      reason:  calendar_interval introduced in 7.2.0
 
   - do:
       indices.create: