add support for missing_bucket on Composite Aggregation (#3420)

per elastic/elasticsearch#29465

Author: Mpdreamz

src/Nest/Aggregations/Bucket/Composite/CompositeAggregationSource.cs

@@ -37,6 +37,13 @@ public interface ICompositeAggregationSource
 		/// </summary>
 		[JsonProperty("order")]
 		SortOrder? Order { get; set; }
+
+		/// <summary>
+		/// By default documents without a value for a given source are ignored. It is possible to include
+		/// them in the response as null by setting this to true
+		/// </summary>
+		[JsonProperty("missing_bucket")]
+		bool? MissingBucket { get; set; }
 	}
 
 	/// <inheritdoc />
@@ -59,6 +66,9 @@ protected CompositeAggregationSourceBase(string name) =>
 
 		/// <inheritdoc />
 		public SortOrder? Order { get; set; }
+
+		/// <inheritdoc />
+		public bool? MissingBucket { get; set; }
 	}
 
 	/// <inheritdoc cref="ICompositeAggregationSource"/>
@@ -93,6 +103,7 @@ public abstract class CompositeAggregationSourceDescriptorBase<TDescriptor, TInt
 		string ICompositeAggregationSource.SourceType => _sourceType;
 		Field ICompositeAggregationSource.Field { get; set; }
 		SortOrder? ICompositeAggregationSource.Order { get; set; }
+		bool? ICompositeAggregationSource.MissingBucket { get; set; }
 
 		protected CompositeAggregationSourceDescriptorBase(string name, string sourceType)
 		{
@@ -108,6 +119,9 @@ protected CompositeAggregationSourceDescriptorBase(string name, string sourceTyp
 
 		/// <inheritdoc cref="ICompositeAggregationSource.Order"/>
 		public TDescriptor Order(SortOrder? order) => Assign(a => a.Order = order);
+
+		/// <inheritdoc cref="ICompositeAggregationSource.MissingBucket"/>
+		public TDescriptor MissingBucket(bool? includeMissing = true) => Assign(a => a.MissingBucket = includeMissing);
 	}
 
 	internal class CompositeAggregationSourceConverter : ReserializeJsonConverter<CompositeAggregationSourceBase, ICompositeAggregationSource>

src/Tests/Tests.Domain/Project.cs

@@ -57,7 +57,7 @@ public class Project
 				.RuleFor(p => p.NumberOfCommits, f => Gimme.Random.Number(1, 1000))
 				.RuleFor(p => p.NumberOfContributors, f => Gimme.Random.Number(1, 200))
 				.RuleFor(p => p.Ranges, f => Ranges.Generator.Generate())
-				.RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List<string> { "master", "dev", "release", "qa", "test" }, 2))
+				.RuleFor(p => p.Branches, f => Gimme.Random.ListItems(new List<string> { "master", "dev", "release", "qa", "test" }))
 				.RuleFor(p => p.SourceOnly, f =>
 					TestConfiguration.Instance.Random.SourceSerializer ? new SourceOnlyObject() : null
 				)

src/Tests/Tests/Aggregations/Bucket/Composite/CompositeAggregationUsageTests.cs

@@ -155,7 +155,7 @@ public CompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage usage) :
 			};
 
 		/**==== Handling Responses
-		 * Each Composite aggregation bucket key is an `CompositeKey`, a specialized
+		 * Each Composite aggregation bucket key is a `CompositeKey` type, a specialized
 		 * `IReadOnlyDictionary<string, object>` type with methods to convert values to supported types
 		 */
 		protected override void ExpectResponse(ISearchResponse<Project> response)
@@ -196,6 +196,130 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 		}
 	}
 
+	/**[float]
+	* == Missing buckets
+	* By default documents without a value for a given source are ignored.
+	* It is possible to include them in the response by setting missing_bucket to `true` (defaults to `false`):
+	*
+	* NOTE: Only available in Elasticsearch 6.4.0+
+	*/
+	[SkipVersion("<6.4.0", "Missing buckets added to Composite Aggregation Elasticsearch 6.4.0+")]
+	public class CompositeAggregationMissingBucketUsageTests : ProjectsOnlyAggregationUsageTestBase
+	{
+		public CompositeAggregationMissingBucketUsageTests(ReadOnlyCluster i, EndpointUsage usage) : base(i, usage) { }
+
+		protected override object AggregationJson => new
+		{
+			my_buckets = new
+			{
+				composite = new
+				{
+					sources = new object[]
+					{
+						new
+						{
+							branches = new
+							{
+								terms = new
+								{
+									field = "branches.keyword",
+									order = "asc",
+									missing_bucket = true
+								}
+							}
+						},
+					}
+				},
+				aggs = new
+				{
+					project_tags = new
+					{
+						nested = new { path = "tags" },
+						aggs = new
+						{
+							tags = new { terms = new {field = "tags.name"} }
+						}
+					}
+				}
+			}
+		};
+
+		protected override Func<AggregationContainerDescriptor<Project>, IAggregationContainer> FluentAggs => a => a
+			.Composite("my_buckets", date => date
+				.Sources(s => s
+					.Terms("branches", t => t
+						.Field(f => f.Branches.Suffix("keyword"))
+						.MissingBucket()
+						.Order(SortOrder.Ascending)
+					)
+				)
+				.Aggregations(childAggs => childAggs
+					.Nested("project_tags", n => n
+						.Path(p => p.Tags)
+						.Aggregations(nestedAggs => nestedAggs
+							.Terms("tags", avg => avg.Field(p => p.Tags.First().Name))
+						)
+					)
+				)
+			);
+
+		protected override AggregationDictionary InitializerAggs =>
+			new CompositeAggregation("my_buckets")
+			{
+				Sources = new List<ICompositeAggregationSource>
+				{
+					new TermsCompositeAggregationSource("branches")
+					{
+						Field = Infer.Field<Project>(f => f.Branches.Suffix("keyword")),
+						MissingBucket = true,
+						Order = SortOrder.Ascending
+					}
+				},
+				Aggregations = new NestedAggregation("project_tags")
+				{
+					Path = Field<Project>(p => p.Tags),
+					Aggregations = new TermsAggregation("tags")
+					{
+						Field = Field<Project>(p => p.Tags.First().Name)
+					}
+				}
+			};
+
+		/**==== Handling Responses
+		 * Each Composite aggregation bucket key is an `CompositeKey`, a specialized
+		 * `IReadOnlyDictionary<string, object>` type with methods to convert values to supported types
+		 */
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			response.ShouldBeValid();
+
+			var composite = response.Aggregations.Composite("my_buckets");
+			composite.Should().NotBeNull();
+			composite.Buckets.Should().NotBeNullOrEmpty();
+			composite.AfterKey.Should().NotBeNull();
+
+			if (TestConfiguration.Instance.InRange(">=6.3.0"))
+				composite.AfterKey.Should().HaveCount(1).And.ContainKeys("branches");
+
+			var i = 0;
+			foreach (var item in composite.Buckets)
+			{
+				var key = item.Key;
+				key.Should().NotBeNull();
+
+				key.TryGetValue("branches", out string branches).Should().BeTrue("expected to find 'branches' in composite bucket");
+				if (i == 0) branches.Should().BeNull("First key should be null as we expect to have some projects with no branches");
+				else branches.Should().NotBeNullOrEmpty();
+
+				var nested = item.Nested("project_tags");
+				nested.Should().NotBeNull();
+
+				var nestedTerms = nested.Terms("tags");
+				nestedTerms.Buckets.Count.Should().BeGreaterThan(0);
+				i++;
+			}
+		}
+	}
 
 	//hide
 	[SkipVersion("<6.3.0", "Date histogram source only supports format starting from Elasticsearch 6.3.0+")]
@@ -286,10 +410,6 @@ public DateFormatCompositeAggregationUsageTests(ReadOnlyCluster i, EndpointUsage
 				}
 			};
 
-		/**==== Handling Responses
-		 * Each Composite aggregation bucket key is an `CompositeKey`, a specialized
-		 * `IReadOnlyDictionary<string, object>` type with methods to convert values to supported types
-		 */
 		protected override void ExpectResponse(ISearchResponse<Project> response)
 		{
 			response.ShouldBeValid();