diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 9a174ac7221e0..fd6d3b442462f 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -16,6 +16,8 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -81,13 +83,66 @@ public Object extract(SearchHit hit) { Object value = null; DocumentField field = null; if (hitName != null) { - // a nested field value is grouped under the nested parent name (ie dep.dep_name lives under "dep":[{dep_name:value}]) - field = hit.field(hitName); + value = unwrapFieldsMultiValue(extractNestedField(hit)); } else { field = hit.field(fieldName); + if (field != null) { + value = unwrapFieldsMultiValue(field.getValues()); + } + } + return value; + } + + /* + * For a path of fields like root.nested1.nested2.leaf where nested1 and nested2 are nested field types, + * fieldName is root.nested1.nested2.leaf, while hitName is root.nested1.nested2 + * We first look for root.nested1.nested2 or root.nested1 or root in the SearchHit until we find something. + * If the DocumentField lives under "root.nested1" the remaining path to search for (in the DocumentField itself) is nested2. + * After this step is done, what remains to be done is just getting the leaf values. + */ + @SuppressWarnings("unchecked") + private Object extractNestedField(SearchHit hit) { + Object value; + DocumentField field; + String tempHitname = hitName; + List remainingPath = new ArrayList<>(); + // first, search for the "root" DocumentField under which the remaining path of nested document values is + while ((field = hit.field(tempHitname)) == null) { + int indexOfDot = tempHitname.lastIndexOf("."); + if (indexOfDot < 0) {// there is no such field in the hit + return null; + } + remainingPath.add(0, tempHitname.substring(indexOfDot + 1)); + tempHitname = tempHitname.substring(0, indexOfDot); } - if (field != null) { - value = unwrapFieldsMultiValue(field.getValues()); + // then dig into DocumentField's structure until we reach the field we are interested into + if (remainingPath.size() > 0) { + List values = field.getValues(); + Iterator pathIterator = remainingPath.iterator(); + while (pathIterator.hasNext()) { + String pathElement = pathIterator.next(); + Map> elements = (Map>) values.get(0); + values = elements.get(pathElement); + /* + * if this path is not found it means we hit another nested document (inner_root_1.inner_root_2.nested_field_2) + * something like this + * "root_field_1.root_field_2.nested_field_1" : [ + * { + * "inner_root_1.inner_root_2.nested_field_2" : [ + * { + * "leaf_field" : [ + * "abc2" + * ] + * So, start re-building the path until the right one is found, ie inner_root_1.inner_root_2...... + */ + while (values == null) { + pathElement += "." + pathIterator.next(); + values = elements.get(pathElement); + } + } + value = ((Map) values.get(0)).get(fieldName.substring(hitName.length() + 1)); + } else { + value = field.getValues(); } return value; } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index 2a0391290bc1d..52202a8a3a619 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -12,21 +12,25 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.ql.util.Holder; import org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase; import org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase; import java.io.IOException; import java.io.InputStream; import java.sql.JDBCType; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.assertResponse; import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.columnInfo; @@ -59,7 +63,7 @@ public void testTextField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "text_field", "text", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "text_field", "text", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(text))); assertResponse(expected, runSql(query)); } else { @@ -96,7 +100,7 @@ public void testKeywordField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreAbove ? null : keyword))); assertResponse(expected, runSql(query)); } else { @@ -132,10 +136,7 @@ public void testConstantKeywordField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", "constant_keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)) - ); + expected.put("columns", asList(columnInfo("plain", "constant_keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(value))); assertResponse(expected, runSql(query)); } else { @@ -172,7 +173,7 @@ public void testWildcardField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreAbove ? null : wildcard))); assertResponse(expected, runSql(query)); } else { @@ -193,10 +194,7 @@ public void testFractionsForNonFloatingPointTypes() throws IOException { index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE)) - ); + expected.put("columns", asList(columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE))); // because "coerce" is true, a "123.456" floating point number STRING should be converted to 123, no matter the numeric field type expected.put("rows", singletonList(singletonList(123))); @@ -227,10 +225,7 @@ public void testCoerceForFloatingPointTypes() throws IOException { index("{\"" + fieldType + "_field\":\"" + floatingPointNumber + "\"}"); Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE)) - ); + expected.put("columns", asList(columnInfo("plain", fieldType + "_field", fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE))); // because "coerce" is true, a "123.456" floating point number STRING should be converted to 123.456 as number // and converted to 123.5 for "scaled_float" type @@ -308,7 +303,7 @@ private void testField(String fieldType, Object value) throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", fieldName, fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", fieldName, fieldType, jdbcTypeFor(fieldType), Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : actualValue))); assertResponse(expected, runSql(query)); } else { @@ -340,7 +335,7 @@ public void testBooleanField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "boolean_field", "boolean", JDBCType.BOOLEAN, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "boolean_field", "boolean", JDBCType.BOOLEAN, Integer.MAX_VALUE))); // adding the boolean as a String here because parsing the response will yield a "true"/"false" String expected.put("rows", singletonList(singletonList(booleanField))); assertResponse(expected, runSql(query)); @@ -380,7 +375,7 @@ public void testIpField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "ip_field", "ip", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "ip_field", "ip", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : actualValue))); assertResponse(expected, runSql(query)); } else { @@ -419,10 +414,7 @@ public void testGeoPointField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", "geo_point_field", "geo_point", JDBCType.VARCHAR, Integer.MAX_VALUE)) - ); + expected.put("columns", asList(columnInfo("plain", "geo_point_field", "geo_point", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : "POINT (-71.34 41.12)"))); assertResponse(expected, runSql(query)); } else { @@ -460,10 +452,7 @@ public void testGeoShapeField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", "geo_shape_field", "geo_shape", JDBCType.VARCHAR, Integer.MAX_VALUE)) - ); + expected.put("columns", asList(columnInfo("plain", "geo_shape_field", "geo_shape", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : "POINT (-77.03653 38.897676)"))); assertResponse(expected, runSql(query)); } else { @@ -502,7 +491,7 @@ public void testShapeField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "shape_field", "shape", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("columns", asList(columnInfo("plain", "shape_field", "shape", JDBCType.VARCHAR, Integer.MAX_VALUE))); expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : shapeField))); assertResponse(expected, runSql(query)); } else { @@ -532,13 +521,13 @@ public void testAliasFromDocValueField() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", "keyword_field_alias", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", "a.b.c.keyword_field_alias", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(keyword, keyword, keyword))); + expected.put("rows", singletonList(asList(keyword, keyword, keyword))); assertResponse(expected, runSql("SELECT keyword_field, keyword_field_alias, a.b.c.keyword_field_alias FROM test")); } @@ -564,13 +553,13 @@ public void testAliasFromSourceField() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", "text_field", "text", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", "text_field_alias", "text", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", "a.b.c.text_field_alias", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(text, text, text))); + expected.put("rows", singletonList(asList(text, text, text))); assertResponse(expected, runSql("SELECT text_field, text_field_alias, a.b.c.text_field_alias FROM test")); } @@ -596,13 +585,13 @@ public void testAliasAggregatableFromSourceField() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", "integer_field", "integer", JDBCType.INTEGER, Integer.MAX_VALUE), columnInfo("plain", "integer_field_alias", "integer", JDBCType.INTEGER, Integer.MAX_VALUE), columnInfo("plain", "a.b.c.integer_field_alias", "integer", JDBCType.INTEGER, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(number, number, number))); + expected.put("rows", singletonList(asList(number, number, number))); assertResponse(expected, runSql("SELECT integer_field, integer_field_alias, a.b.c.integer_field_alias FROM test")); } @@ -644,13 +633,13 @@ public void testTextFieldWithKeywordSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "text", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(text, ignoreAbove ? null : text))); + expected.put("rows", singletonList(asList(text, ignoreAbove ? null : text))); assertResponse(expected, runSql(query)); } else { expectSourceDisabledError(query); @@ -699,15 +688,15 @@ public void testTextFieldWithIntegerNumberSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "text", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", subFieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE) ) ); if (ignoreMalformed) { - expected.put("rows", singletonList(Arrays.asList("foo", null))); + expected.put("rows", singletonList(asList("foo", null))); } else { - expected.put("rows", singletonList(Arrays.asList(String.valueOf(number), number))); + expected.put("rows", singletonList(asList(String.valueOf(number), number))); } assertResponse(expected, runSql(query)); } else { @@ -758,15 +747,15 @@ public void testTextFieldWithIpSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "text", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", subFieldName, "ip", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); if (ignoreMalformed) { - expected.put("rows", singletonList(Arrays.asList("foo", null))); + expected.put("rows", singletonList(asList("foo", null))); } else { - expected.put("rows", singletonList(Arrays.asList(ip, ip))); + expected.put("rows", singletonList(asList(ip, ip))); } assertResponse(expected, runSql(query)); } else { @@ -824,15 +813,15 @@ public void testNumberFieldWithTextOrKeywordSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE), columnInfo("plain", subFieldName, isKeyword ? "keyword" : "text", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); if (ignoreMalformed) { - expected.put("rows", singletonList(Arrays.asList(null, "foo"))); + expected.put("rows", singletonList(asList(null, "foo"))); } else { - expected.put("rows", singletonList(Arrays.asList(number, String.valueOf(number)))); + expected.put("rows", singletonList(asList(number, String.valueOf(number)))); } assertResponse(expected, runSql(query)); } else { @@ -894,15 +883,15 @@ public void testIpFieldWithTextOrKeywordSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "ip", JDBCType.VARCHAR, Integer.MAX_VALUE), columnInfo("plain", subFieldName, isKeyword ? "keyword" : "text", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); if (ignoreMalformed) { - expected.put("rows", singletonList(Arrays.asList(null, "foo"))); + expected.put("rows", singletonList(asList(null, "foo"))); } else { - expected.put("rows", singletonList(Arrays.asList(ip, ip))); + expected.put("rows", singletonList(asList(ip, ip))); } assertResponse(expected, runSql(query)); } else { @@ -959,14 +948,14 @@ public void testIntegerFieldWithByteSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE), columnInfo("plain", subFieldName, "byte", JDBCType.TINYINT, Integer.MAX_VALUE) ) ); if (explicitSourceSetting == false || enableSource) { if (isByte || subFieldIgnoreMalformed) { - expected.put("rows", singletonList(Arrays.asList(number, isByte ? number : null))); + expected.put("rows", singletonList(asList(number, isByte ? number : null))); } else { expected.put("rows", Collections.emptyList()); } @@ -1028,14 +1017,14 @@ public void testByteFieldWithIntegerSubfield() throws IOException { Map expected = new HashMap<>(); expected.put( "columns", - Arrays.asList( + asList( columnInfo("plain", fieldName, "byte", JDBCType.TINYINT, Integer.MAX_VALUE), columnInfo("plain", subFieldName, "integer", JDBCType.INTEGER, Integer.MAX_VALUE) ) ); if (explicitSourceSetting == false || enableSource) { if (isByte || rootIgnoreMalformed) { - expected.put("rows", singletonList(Arrays.asList(isByte ? number : null, number))); + expected.put("rows", singletonList(asList(isByte ? number : null, number))); } else { expected.put("rows", Collections.emptyList()); } @@ -1050,6 +1039,375 @@ public void testByteFieldWithIntegerSubfield() throws IOException { } } + public void testNestedFieldsHierarchyWithMultiNestedValues() throws IOException { + Request request = new Request("PUT", "/test"); + request.setJsonEntity( + "{" + + " \"mappings\" : {" + + " \"properties\" : {" + + " \"h\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"i\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"j\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"f\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"o\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"b\" : {" + + " \"properties\" : {" + + " \"a\" : {" + + " \"type\" : \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}" + ); + client().performRequest(request); + index("{\"h\": [{\"i\":\"123\", \"j\":\"abc\"}, {\"i\":\"890\", \"j\":\"xyz\"}, {\"i\":\"567\", \"j\":\"klm\"}],\"test\":\"foo\"}"); + + Map expected = new HashMap<>(); + expected.put( + "columns", + asList( + columnInfo("plain", "h.i", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "h.j", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) + ) + ); + expected.put("rows", asList(asList("123", "abc", "foo"), asList("890", "xyz", "foo"), asList("567", "klm", "foo"))); + assertResponse(expected, runSql("SELECT h.i, h.j, test FROM test")); + } + + public void testNestedFieldsHierarchyWithMissingValue() throws IOException { + Request request = new Request("PUT", "/test"); + request.setJsonEntity( + "{" + + " \"mappings\" : {" + + " \"properties\" : {" + + " \"h\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"i\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"f\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"o\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"b\" : {" + + " \"properties\" : {" + + " \"a\" : {" + + " \"type\" : \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}" + ); + client().performRequest(request); + index("{\"h\": [{\"f\":{\"b\": {\"a\": \"ABC\"}}}]}"); + + Map expected = new HashMap<>(); + expected.put("columns", singletonList(columnInfo("plain", "h.f.o", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList(null))); + assertResponse(expected, runSql("SELECT h.f.o FROM test")); + + expected.put("columns", singletonList(columnInfo("plain", "h.i", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + assertResponse(expected, runSql("SELECT h.i FROM test")); + } + + public void testNestedFieldsHierarchyExtractDeeplyNestedValue() throws IOException { + Request request = new Request("PUT", "/test"); + request.setJsonEntity( + "{" + + " \"mappings\" : {" + + " \"properties\" : {" + + " \"h\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"i\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"f\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"o\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"b\" : {" + + " \"properties\" : {" + + " \"a\" : {" + + " \"type\" : \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}" + ); + client().performRequest(request); + index("{\"h\": [{\"f\":{\"b\": {\"a\": \"ABC\"}}}]}"); + + Map expected = new HashMap<>(); + expected.put("columns", singletonList(columnInfo("plain", "h.f.b.a", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList("ABC"))); + assertResponse(expected, runSql("SELECT h.f.b.a FROM test")); + } + + public void testNestedFieldsHierarchyWithArrayOfValues() throws IOException { + Request request = new Request("PUT", "/test"); + request.setJsonEntity( + "{" + + " \"mappings\" : {" + + " \"properties\" : {" + + " \"h\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"i\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"j\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"f\" : {" + + " \"type\" : \"nested\"," + + " \"properties\" : {" + + " \"o\" : {" + + " \"type\" : \"keyword\"" + + " }," + + " \"b\" : {" + + " \"properties\" : {" + + " \"a\" : {" + + " \"type\" : \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + " }" + + "}" + ); + client().performRequest(request); + index( + "{\"h\": [{\"i\":[\"123\",\"124\",\"125\"], \"j\":\"abc\"}, {\"i\":\"890\", \"j\":\"xyz\"}, {\"i\":\"567\", \"j\":\"klm\"}]," + + "\"test\":\"foo\"}" + ); + + Map expected = new HashMap<>(); + Map actual = new HashMap<>(); + expected.put( + "columns", + asList( + columnInfo("plain", "h.i", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "h.j", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE), + columnInfo("plain", "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) + ) + ); + expected.put("rows", asList(asList("123", "abc", "foo"), asList("890", "xyz", "foo"), asList("567", "klm", "foo"))); + Request sqlRequest = new Request("POST", RestSqlTestCase.SQL_QUERY_REST_ENDPOINT); + sqlRequest.addParameter("error_trace", "true"); + sqlRequest.addParameter("pretty", "true"); + sqlRequest.setEntity( + new StringEntity( + query("SELECT h.i, h.j, test FROM test").mode("plain").fieldMultiValueLeniency(true).toString(), + ContentType.APPLICATION_JSON + ) + ); + Response response = client().performRequest(sqlRequest); + try (InputStream content = response.getEntity().getContent()) { + actual = XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + } + assertResponse(expected, actual); + } + + /* + * From a randomly created mapping using "object" field types and "nested" field types like the one below, we look at + * extracting the values from the deepest "nested" field type. + * The query to use for the mapping below would be "SELECT HETeC.fdeuk.oDwgT FROM test" + * { + * "mappings" : { + * "properties" : { + * "HETeC" : { + * "type" : "nested", + * "properties" : { + * "iBtgB" : { + * "type" : "keyword" + * }, + * "fdeuk" : { + * "type" : "nested", + * "properties" : { + * "oDwgT" : { + * "type" : "keyword" + * }, + * "biXlb" : { + * "properties" : { + * "AlkJR" : { + * "type" : "keyword" + * } + * } + * } + * } + * } + * } + * } + * } + * } + * } + */ + public void testNestedFieldsHierarchy() throws IOException { + final int minDepth = 2; + final int maxDepth = 6; + final int depth = between(minDepth, maxDepth); + + Request request = new Request("PUT", "/test"); + XContentBuilder index = JsonXContent.contentBuilder().prettyPrint().startObject(); + List> path = new ArrayList<>(depth); + StringBuilder bulkContent = new StringBuilder(); + Holder randomValue = new Holder<>(""); + index.startObject("mappings"); + { + index.startObject("properties"); + { + addField(index, false, depth, path, bulkContent, randomValue); + } + index.endObject(); + } + index.endObject(); + index.endObject(); + + request.setJsonEntity(Strings.toString(index)); + client().performRequest(request); + index("{" + bulkContent.toString() + "}"); + + // the path ends with either a NESTED field or an OBJECT field (both having a leaf field as a sub-field) + // if it's nested, we use this field + // if it's object, we need to strip every field starting from the end until we reach a nested field + int endOfPathIndex = path.size() - 2; // -1 because we skip the leaf field at the end and another -1 because it's 0-based + while (path.get(endOfPathIndex--).v2() != NestedFieldType.NESTED) { + } // find the first nested field starting from the end + + StringBuilder stringPath = new StringBuilder(path.get(0).v1()); // the path we will ask for in the sql query + for (int i = 1; i <= endOfPathIndex + 2; i++) { // +2 because the index is now at the [index_of_a_nested_field]-1 + if (path.get(i).v2() != NestedFieldType.LEAF || i == endOfPathIndex + 2) { + stringPath.append("."); + stringPath.append(path.get(i).v1()); + } + } + + Map expected = new HashMap<>(); + expected.put("columns", singletonList(columnInfo("plain", stringPath.toString(), "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList(randomValue.get()))); + assertResponse(expected, runSql("SELECT " + stringPath.toString() + " FROM test")); + } + + private enum NestedFieldType { + NESTED, + OBJECT, + LEAF; + } + + private void addField( + XContentBuilder index, + boolean nestedFieldAdded, + int remainingFields, + List> path, + StringBuilder bulkContent, + Holder randomValue + ) throws IOException { + String fieldName = randomAlphaOfLength(5); + String leafFieldName = randomAlphaOfLength(5); + + // we need to make sure we add at least one nested field to the mapping, otherwise the test is not about nested fields + if (shouldAddNestedField() || (nestedFieldAdded == false && remainingFields == 1)) { + path.add(new Tuple(fieldName, NestedFieldType.NESTED)); + path.add(new Tuple(leafFieldName, NestedFieldType.LEAF)); + index.startObject(fieldName); + { + index.field("type", "nested"); + index.startObject("properties"); + { + // A nested field always has a leaf field, even if not all nested fields in a path will have this value + // indexed. We will index only the "leaf" field of the last nested field in the path, because this is the + // one we will ask back from ES + index.startObject(leafFieldName); + { + index.field("type", "keyword"); + } + index.endObject(); + // from time to time set a null value instead of an actual value + if (rarely()) { + randomValue.set(null); + bulkContent.append("\"" + fieldName + "\":{\"" + leafFieldName + "\":null"); + } else { + randomValue.set(randomAlphaOfLength(10)); + bulkContent.append("\"" + fieldName + "\":{\"" + leafFieldName + "\":\"" + randomValue.get() + "\""); + } + if (remainingFields > 1) { + bulkContent.append(","); + addField(index, true, remainingFields - 1, path, bulkContent, randomValue); + } + bulkContent.append("}"); + } + index.endObject(); + } + index.endObject(); + } else { + path.add(new Tuple(fieldName, NestedFieldType.OBJECT)); + index.startObject(fieldName); + index.startObject("properties"); + { + bulkContent.append("\"" + fieldName + "\":{"); + // if this is the last field in the mapping and it's non-nested, add a keyword to it, otherwise the mapping + // is incomplete and an error will be thrown at mapping creation time + if (remainingFields == 1) { + path.add(new Tuple(leafFieldName, NestedFieldType.LEAF)); + index.startObject(leafFieldName); + { + index.field("type", "keyword"); + } + index.endObject(); + bulkContent.append("\"" + leafFieldName + "\":\"" + randomAlphaOfLength(10) + "\""); + } else { + addField(index, nestedFieldAdded, remainingFields - 1, path, bulkContent, randomValue); + } + bulkContent.append("}"); + } + index.endObject(); + index.endObject(); + } + } + + private boolean shouldAddNestedField() { + return randomBoolean(); + } + private void expectSourceDisabledError(String query) { expectBadRequest(() -> { client().performRequest(buildRequest(query)); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java index 71f59751d285e..bcb0c5d7a2dfa 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java @@ -28,6 +28,7 @@ import static org.elasticsearch.xpack.sql.proto.Protocol.COLUMNAR_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.CURSOR_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.FETCH_SIZE_NAME; +import static org.elasticsearch.xpack.sql.proto.Protocol.FIELD_MULTI_VALUE_LENIENCY_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.FILTER_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.MODE_NAME; import static org.elasticsearch.xpack.sql.proto.Protocol.PARAMS_NAME; @@ -105,6 +106,11 @@ public RequestObjectBuilder binaryFormat(Boolean binaryFormat) { return this; } + public RequestObjectBuilder fieldMultiValueLeniency(Boolean fieldMultiValueLeniency) { + request.append(field(FIELD_MULTI_VALUE_LENIENCY_NAME, fieldMultiValueLeniency)); + return this; + } + private static String field(String name, Object value) { if (value == null) { return StringUtils.EMPTY; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index e0b1d081ec322..e503a6649ae02 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -338,7 +338,7 @@ private Tuple nestedHitFieldRef(FieldAttribute SqlDataTypes.format(attr.field().getDataType()), SqlDataTypes.isFromDocValuesOnly(attr.field().getDataType())); - SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), attr.parent().name()); + SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), attr.nestedParent().name()); return new Tuple<>( new QueryContainer(q, aggs, fields, aliases, pseudoFunctions, scalarFunctions, sort, limit, trackHits, includeFrozen,