Support 2nd level field collapsing as per elastic/elasticsearch#31808

Author: Mpdreamz

src/Nest/Search/Search/Collapsing/FieldCollapse.cs

@@ -27,38 +27,52 @@ public interface IFieldCollapse
 		[JsonProperty("inner_hits")]
 		IInnerHits InnerHits { get; set; }
 
+		/// <summary>
+		/// The expansion of the group is done by sending an additional query for each inner_hit request for each collapsed hit returned
+		/// in the response. This can significantly slow things down if you have too many groups and/or inner_hit requests.
+		/// The max_concurrent_group_searches request parameter can be used to control the maximum number of
+		/// concurrent searches allowed in this phase. The default is based on the number of data nodes and the
+		/// default search thread pool size.
+		/// </summary>
 		[JsonProperty("max_concurrent_group_searches")]
 		int? MaxConcurrentGroupSearches { get; set; }
 	}
 
-	/// <inheritdoc/>
+	/// <inheritdoc cref="IFieldCollapse"/>
 	public class FieldCollapse : IFieldCollapse
 	{
-		/// <inheritdoc/>
+		/// <inheritdoc cref="IFieldCollapse.Field"/>
 		public Field Field { get; set; }
 
+		/// <inheritdoc cref="IFieldCollapse.InnerHits"/>
 		public IInnerHits InnerHits { get; set; }
 
+		/// <inheritdoc cref="IFieldCollapse.MaxConcurrentGroupSearches"/>
 		public int? MaxConcurrentGroupSearches { get; set; }
+
 	}
 
-	/// <inheritdoc/>
+	/// <inheritdoc cref="IFieldCollapse"/>
 	public class FieldCollapseDescriptor<T> : DescriptorBase<FieldCollapseDescriptor<T>, IFieldCollapse>, IFieldCollapse
 		where T : class
 	{
-		/// <inheritdoc/>
 		Field IFieldCollapse.Field { get; set; }
 		IInnerHits IFieldCollapse.InnerHits { get; set; }
 		int? IFieldCollapse.MaxConcurrentGroupSearches { get; set; }
 
+		/// <inheritdoc cref="IFieldCollapse.MaxConcurrentGroupSearches"/>
 		public FieldCollapseDescriptor<T> MaxConcurrentGroupSearches(int? maxConcurrentGroupSearches) =>
 			Assign(a => a.MaxConcurrentGroupSearches = maxConcurrentGroupSearches);
 
+		/// <inheritdoc cref="IFieldCollapse.Field"/>
 		public FieldCollapseDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
 
+		/// <inheritdoc cref="IFieldCollapse.Field"/>
 		public FieldCollapseDescriptor<T> Field(Expression<Func<T, object>> objectPath) => Assign(a => a.Field = objectPath);
 
+		/// <inheritdoc cref="IFieldCollapse.InnherHits"/>
 		public FieldCollapseDescriptor<T> InnerHits(Func<InnerHitsDescriptor<T>, IInnerHits> selector = null) =>
 			Assign(a => a.InnerHits = selector.InvokeOrDefault(new InnerHitsDescriptor<T>()));
+
 	}
 }

src/Nest/Search/Search/InnerHits/InnerHits.cs

@@ -41,6 +41,12 @@ public interface IInnerHits
 
 		[JsonProperty("ignore_unmapped")]
 		bool? IgnoreUnmapped { get; set; }
+
+		/// <summary>
+		/// Provides a second level of collapsing, NOTE: Elasticsearch only supports collapsing up to two levels.
+		/// </summary>
+		[JsonProperty("collapse")]
+		IFieldCollapse Collapse { get; set; }
 	}
 
 	public class InnerHits : IInnerHits
@@ -66,6 +72,9 @@ public class InnerHits : IInnerHits
 		public Fields DocValueFields { get; set; }
 
 		public bool? IgnoreUnmapped { get; set; }
+
+		/// <inheritdoc cref="IInnerHits.Collapse"/>
+		public IFieldCollapse Collapse { get; set; }
 	}
 
 	[JsonObject(MemberSerialization = MemberSerialization.OptIn)]
@@ -82,6 +91,7 @@ public class InnerHitsDescriptor<T> : DescriptorBase<InnerHitsDescriptor<T>, IIn
 		IScriptFields IInnerHits.ScriptFields { get; set; }
 		Fields IInnerHits.DocValueFields { get; set; }
 		bool? IInnerHits.IgnoreUnmapped { get; set; }
+		IFieldCollapse IInnerHits.Collapse { get; set; }
 
 		public InnerHitsDescriptor<T> From(int? from) => Assign(a => a.From = from);
 
@@ -115,5 +125,9 @@ public InnerHitsDescriptor<T> DocValueFields(Func<FieldsDescriptor<T>, IPromise<
 		public InnerHitsDescriptor<T> DocValueFields(Fields fields) => Assign(a => a.DocValueFields = fields);
 
 		public InnerHitsDescriptor<T> IgnoreUnmapped(bool? ignoreUnmapped = true) => Assign(a => a.IgnoreUnmapped = ignoreUnmapped);
+
+		/// <inheritdoc cref="IInnerHits.Collapse"/>
+		public InnerHitsDescriptor<T> Collapse(Func<FieldCollapseDescriptor<T>, IFieldCollapse> collapseSelector) =>
+			Assign(a => a.Collapse = collapseSelector?.Invoke(new FieldCollapseDescriptor<T>()));
 	}
 }

src/Nest/Search/Search/SearchRequest.cs

@@ -399,6 +399,7 @@ public SearchDescriptor<T> Highlight(Func<HighlightDescriptor<T>, IHighlight> hi
 		/// For instance the query below retrieves the best tweet for each user and sorts them by number of likes.
 		/// <para>
 		/// NOTE: The collapsing is applied to the top hits only and does not affect aggregations.
+		/// You can only collapse to a depth of 2.
 		/// </para>
 		/// </summary>
 		public SearchDescriptor<T> Collapse(Func<FieldCollapseDescriptor<T>, IFieldCollapse> collapseSelector) =>

src/Tests/Tests.Configuration/tests.default.yaml

@@ -5,7 +5,7 @@
 # tracked by git).
 
 # mode either u (unit test), i (integration test) or m (mixed mode)
-mode: u
+mode: m
 # the elasticsearch version that should be started
 # Can be a snapshot version of sonatype or "latest" to get the latest snapshot of sonatype
 elasticsearch_version: 6.4.1

src/Tests/Tests/Search/Search/Collapsing/FieldCollapseUsageTests.cs

@@ -1,19 +1,16 @@
 using System;
+using Elastic.Xunit.XunitPlumbing;
 using FluentAssertions;
 using Nest;
 using Tests.Core.Extensions;
 using Tests.Core.ManagedElasticsearch.Clusters;
 using Tests.Core.ManagedElasticsearch.NodeSeeders;
 using Tests.Domain;
 using Tests.Framework.Integration;
-using Tests.Framework.ManagedElasticsearch.Clusters;
-using Tests.Framework.ManagedElasticsearch.NodeSeeders;
 using static Nest.Infer;
 
 namespace Tests.Search.Search.Collapsing
 {
-	/**
-	 */
 	public class FieldCollapseUsageTests : SearchUsageTestBase
 	{
 		protected override string UrlPath => $"/{DefaultSeeder.ProjectsAliasFilter}/doc/_search";
@@ -77,4 +74,93 @@ protected override void ExpectResponse(ISearchResponse<Project> response)
 			}
 		}
 	}
+
+	[SkipVersion("<6.4.0", "2nd level collapsing is a new feature in 6.4.0")]
+	public class FieldCollapseSecondLevelUsageTests : SearchUsageTestBase
+	{
+		protected override string UrlPath => $"/{DefaultSeeder.ProjectsAliasFilter}/doc/_search";
+
+		public FieldCollapseSecondLevelUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
+
+		protected override object ExpectJson => new
+		{
+			_source = new { excludes = new [] { "*" } },
+			collapse = new {
+				field = "state",
+				inner_hits = new {
+					_source = new {
+						excludes = new [] { "*" }
+					},
+					collapse = new {
+						field = "name"
+					},
+					from = 1,
+					name = "stateofbeing",
+					size = 5
+				},
+				max_concurrent_group_searches = 1000
+			}
+		};
+
+		protected override Func<SearchDescriptor<Project>, ISearchRequest> Fluent => s => s
+			.Source(source=>source.ExcludeAll())
+			.Index(DefaultSeeder.ProjectsAliasFilter)
+			.Collapse(c => c
+				.Field(f => f.State)
+				.MaxConcurrentGroupSearches(1000)
+				.InnerHits(i => i
+					.Source(source=>source.ExcludeAll())
+					.Name(nameof(StateOfBeing).ToLowerInvariant())
+					.Size(5)
+					.From(1)
+					.Collapse(c2 => c2
+						.Field(p=>p.Name)
+					)
+				)
+			);
+
+		protected override SearchRequest<Project> Initializer => new SearchRequest<Project>(DefaultSeeder.ProjectsAliasFilter)
+		{
+			Source = SourceFilter.ExcludeAll,
+			Collapse = new FieldCollapse
+			{
+				Field = Field<Project>(p => p.State),
+				MaxConcurrentGroupSearches = 1000,
+				InnerHits = new InnerHits
+				{
+					Source = SourceFilter.ExcludeAll,
+					Name = nameof(StateOfBeing).ToLowerInvariant(),
+					Size = 5,
+					From = 1,
+					Collapse = new FieldCollapse
+					{
+						Field = Field<Project>(p=>p.Name)
+					}
+				}
+			}
+		};
+
+		protected override void ExpectResponse(ISearchResponse<Project> response)
+		{
+			var numberOfStates = Enum.GetValues(typeof(StateOfBeing)).Length;
+			response.HitsMetadata.Total.Should().BeGreaterThan(numberOfStates);
+			response.Hits.Count.Should().Be(numberOfStates);
+			foreach (var hit in response.Hits)
+			{
+				var name = nameof(StateOfBeing).ToLowerInvariant();
+				hit.InnerHits.Should().NotBeNull().And.ContainKey(name);
+				var innerHits = hit.InnerHits[name];
+				innerHits.Hits.Total.Should().BeGreaterThan(0);
+				var i = 0;
+				foreach (var innerHit in innerHits.Hits.Hits)
+				{
+					i++;
+					innerHit.Fields.Should().NotBeEmpty()
+						.And.ContainKey("name");
+				}
+
+				i.Should().NotBe(0, "we expect to inspect 2nd level collapsed fields");
+			}
+		}
+	}
 }