Allow max_children only in top level nested sort (#46731)

This commit restricts the usage of max_children to the top level nested sort since it is ignored on the other levels.

Author: erayarslan

server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java

@@ -409,16 +409,15 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
                         throw new QueryShardException(context,
                             "max_children is only supported on last level of nested sort");
                     }
-                    // new nested sorts takes priority
+                    validateMaxChildrenExistOnlyInTopLevelNestedSort(context, nestedSort);
                     nested = resolveNested(context, nestedSort);
                 } else {
                     nested = resolveNested(context, nestedPath, nestedFilter);
                 }
             }
-
             IndexFieldData<?> fieldData = context.getForField(fieldType);
             if (fieldData instanceof IndexNumericFieldData == false
-                    && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
+                && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
                 throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
             }
             final SortField field;
@@ -437,6 +436,18 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         }
     }
 
+    /**
+     * Throws an exception if max children is not located at top level nested sort.
+     */
+    static void validateMaxChildrenExistOnlyInTopLevelNestedSort(QueryShardContext context, NestedSortBuilder nestedSort) {
+        for (NestedSortBuilder child = nestedSort.getNestedSort(); child != null; child = child.getNestedSort()) {
+            if (child.getMaxChildren() != Integer.MAX_VALUE) {
+                throw new QueryShardException(context,
+                    "max_children is only supported on top level of nested sort");
+            }
+        }
+    }
+
     @Override
     public boolean equals(Object other) {
         if (this == other) {

server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java

@@ -66,6 +66,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
+import static org.elasticsearch.search.sort.FieldSortBuilder.validateMaxChildrenExistOnlyInTopLevelNestedSort;
 import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;
 
 /**
@@ -630,6 +631,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
                     "max_children is only supported on last level of nested sort");
             }
             // new nested sorts takes priority
+            validateMaxChildrenExistOnlyInTopLevelNestedSort(context, nestedSort);
             nested = resolveNested(context, nestedSort);
         } else {
             nested = resolveNested(context, nestedPath, nestedFilter);

server/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java

@@ -60,6 +60,7 @@
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.search.sort.FieldSortBuilder.validateMaxChildrenExistOnlyInTopLevelNestedSort;
 import static org.elasticsearch.search.sort.NestedSortBuilder.NESTED_FIELD;
 
 /**
@@ -325,6 +326,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
                     "max_children is only supported on last level of nested sort");
             }
             // new nested sorts takes priority
+            validateMaxChildrenExistOnlyInTopLevelNestedSort(context, nestedSort);
             nested = resolveNested(context, nestedSort);
         } else {
             nested = resolveNested(context, nestedPath, nestedFilter);

server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java

@@ -1421,6 +1421,20 @@ public void testNestedSort() throws IOException, InterruptedException, Execution
                                                             .endObject()
                                                         .endObject()
                                                     .endObject()
+                                                    .startObject("bar")
+                                                        .field("type", "nested")
+                                                        .startObject("properties")
+                                                            .startObject("foo")
+                                                                .field("type", "text")
+                                                                .field("fielddata", true)
+                                                                .startObject("fields")
+                                                                    .startObject("sub")
+                                                                        .field("type", "keyword")
+                                                                    .endObject()
+                                                                .endObject()
+                                                            .endObject()
+                                                        .endObject()
+                                                    .endObject()
                                                 .endObject()
                                             .endObject()
                                         .endObject()
@@ -1471,6 +1485,22 @@ public void testNestedSort() throws IOException, InterruptedException, Execution
         assertThat(hits[0].getSortValues()[0], is("bar"));
         assertThat(hits[1].getSortValues()[0], is("abc"));
 
+        {
+            SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
+                () -> client().prepareSearch()
+                    .setQuery(matchAllQuery())
+                    .addSort(SortBuilders
+                        .fieldSort("nested.bar.foo")
+                        .setNestedSort(new NestedSortBuilder("nested")
+                            .setNestedSort(new NestedSortBuilder("nested.bar")
+                                .setMaxChildren(1)))
+                        .order(SortOrder.DESC))
+                    .get()
+            );
+            assertThat(exc.toString(),
+                containsString("max_children is only supported on top level of nested sort"));
+        }
+
         // We sort on nested sub field
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())