diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 04ec7a22d1430..6e6e48933cf0c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -17,10 +17,12 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; /** @@ -47,7 +49,7 @@ public Object handle(Object object, String fieldName) { EXTRACT_ARRAY { @Override public Object handle(Object object, String _ignored) { - return object instanceof List ? object : singletonList(object); + return object == null ? emptyList() : (object instanceof List ? object : singletonList(object)); } }; @@ -127,18 +129,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Object extract(SearchHit hit) { - Object value = null; - DocumentField field = null; - if (hitName != null) { - // a nested field value is grouped under the nested parent name (ie dep.dep_name lives under "dep":[{dep_name:value}]) - field = hit.field(hitName); - } else { - field = hit.field(fieldName); - } - if (field != null) { - value = unwrapFieldsMultiValue(field.getValues()); - } - return value; + // hitName: a nested field value is grouped under the nested parent name (ie dep.dep_name lives under "dep":[{dep_name:value}]) + String lookupName = hitName != null ? hitName : fieldName; + DocumentField field = hit.field(lookupName); + return multiValueHandling.handle(field != null ? unwrapFieldsMultiValue(field.getValues()) : null, lookupName); } protected Object unwrapFieldsMultiValue(Object values) { @@ -155,11 +149,16 @@ protected Object unwrapFieldsMultiValue(Object values) { return null; } else { if (isPrimitive(list) == false) { - if (list.size() == 1 || multiValueHandling == MultiValueHandling.EXTRACT_ONE) { - return unwrapFieldsMultiValue(list.get(0)); - } else { - throw new QlIllegalArgumentException("Arrays (returned by [{}]) are not supported", fieldName); + List unwrappedList = new ArrayList<>(); + for (Object o : list) { + Object unwrapped = unwrapFieldsMultiValue(o); + if (unwrapped instanceof List) { + unwrappedList.addAll((List) unwrapped); + } else { + unwrappedList.add(unwrapped); + } } + return unwrappedList; } } } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java index 02e97f9c9606c..a20f6ca4b4df5 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java @@ -48,7 +48,24 @@ public enum EsType implements SQLType { INTERVAL_MINUTE_TO_SECOND(ExtraTypes.INTERVAL_MINUTE_SECOND), GEO_POINT( ExtraTypes.GEOMETRY), GEO_SHAPE( ExtraTypes.GEOMETRY), - SHAPE( ExtraTypes.GEOMETRY); + SHAPE( ExtraTypes.GEOMETRY), + BOOLEAN_ARRAY( Types.ARRAY), + BYTE_ARRAY( Types.ARRAY), + SHORT_ARRAY( Types.ARRAY), + INTEGER_ARRAY( Types.ARRAY), + LONG_ARRAY( Types.ARRAY), + DOUBLE_ARRAY( Types.ARRAY), + FLOAT_ARRAY( Types.ARRAY), + HALF_FLOAT_ARRAY( Types.ARRAY), + SCALED_FLOAT_ARRAY( Types.ARRAY), + KEYWORD_ARRAY( Types.ARRAY), + TEXT_ARRAY( Types.ARRAY), + DATETIME_ARRAY( Types.ARRAY), + IP_ARRAY( Types.ARRAY), + BINARY_ARRAY( Types.ARRAY), + GEO_SHAPE_ARRAY( Types.ARRAY), + GEO_POINT_ARRAY( Types.ARRAY), + SHAPE_ARRAY( Types.ARRAY); private final Integer type; diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcArray.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcArray.java new file mode 100644 index 0000000000000..5f0a4b5aad0c7 --- /dev/null +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcArray.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.jdbc; + +import java.sql.Array; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.baseType; + +public class JdbcArray implements Array { + + private final EsType type; + private final List values; + + public JdbcArray(EsType type, List values) { + this.type = type; + this.values = values; + } + @Override + public String getBaseTypeName() throws SQLException { + return baseType(type).getName(); + } + + @Override + public int getBaseType() throws SQLException { + return baseType(type).getVendorTypeNumber(); + } + + @Override + public Object getArray() throws SQLException { + return values.toArray(); + } + + @Override + public Object getArray(Map> map) throws SQLException { + if (map == null || map.isEmpty()) { + return getArray(); + } + throw new SQLFeatureNotSupportedException("getArray with non-empty Map not supported"); + } + + @Override + public Object getArray(long index, int count) throws SQLException { + if (index < 1 || index > Integer.MAX_VALUE) { + throw new SQLException("Index value [" + index + "] out of range [1, " + Integer.MAX_VALUE + "]"); + } + if (count < 0) { + throw new SQLException("Illegal negative count [" + count + "]"); + } + int index0 = (int) index - 1; // 0-based index + int available = index0 < values.size() ? values.size() - index0 : 0; + return available > 0 ? + Arrays.copyOfRange(values.toArray(), index0, index0 + Math.min(count, available)): + new Object[0]; + } + + @Override + public Object getArray(long index, int count, Map> map) throws SQLException { + if (map == null || map.isEmpty()) { + return getArray(index, count); + } + throw new SQLFeatureNotSupportedException("getArray with non-empty Map not supported"); + } + + @Override + public ResultSet getResultSet() throws SQLException { + throw new SQLFeatureNotSupportedException("Array as ResultSet not supported"); + } + + @Override + public ResultSet getResultSet(Map> map) throws SQLException { + throw new SQLFeatureNotSupportedException("Array as ResultSet not supported"); + } + + @Override + public ResultSet getResultSet(long index, int count) throws SQLException { + throw new SQLFeatureNotSupportedException("Array as ResultSet not supported"); + } + + @Override + public ResultSet getResultSet(long index, int count, Map> map) throws SQLException { + throw new SQLFeatureNotSupportedException("Array as ResultSet not supported"); + } + + @Override + public void free() throws SQLException { + // nop + } +} diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java index c21beefcc4ce9..7835fc7e00232 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java @@ -44,6 +44,7 @@ import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.timeAsMillisSinceEpoch; import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.timeAsTime; import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.timeAsTimestamp; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.isArray; class JdbcResultSet implements ResultSet, JdbcWrapper { @@ -934,7 +935,11 @@ public Clob getClob(int columnIndex) throws SQLException { @Override public Array getArray(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException("Array not supported"); + EsType type = columnType(columnIndex); + if (isArray(type) == false) { + throw new SQLException("Cannot get column [" + columnIndex + "] of type [" + type.getName() + "] as array"); + } + return new JdbcArray(type, (List) getObject(columnIndex)); } @Override @@ -954,7 +959,7 @@ public Clob getClob(String columnLabel) throws SQLException { @Override public Array getArray(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException("Array not supported"); + return getArray(column(columnLabel)); } @Override diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index a4943afd35fe4..33b3ed7117a5c 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -27,8 +27,10 @@ import java.time.Period; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Calendar; import java.util.GregorianCalendar; +import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -46,6 +48,7 @@ import static org.elasticsearch.xpack.sql.jdbc.EsType.TIME; import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asDateTimeField; import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.timeAsTime; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.baseType; /** * Conversion utilities for conversion of JDBC types to Java type and back @@ -263,6 +266,29 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL } case IP: return v.toString(); + case BOOLEAN_ARRAY: + case BYTE_ARRAY: + case SHORT_ARRAY: + case INTEGER_ARRAY: + case LONG_ARRAY: + case DOUBLE_ARRAY: + case FLOAT_ARRAY: + case HALF_FLOAT_ARRAY: + case SCALED_FLOAT_ARRAY: + case KEYWORD_ARRAY: + case TEXT_ARRAY: + case DATETIME_ARRAY: + case IP_ARRAY: + case BINARY_ARRAY: + case GEO_SHAPE_ARRAY: + case GEO_POINT_ARRAY: + case SHAPE_ARRAY: + List values = new ArrayList<>(); + for (Object o : (List) v) { + // null value expects a NULL type in the converter + values.add(o == null ? null : convert(o, baseType(columnType), typeString.substring(0, "_ARRAY".length()))); + } + return values; default: throw new SQLException("Unexpected column type [" + typeString + "]"); diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java index a1ac2b45227a5..5db4ee54a66ab 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeUtils.java @@ -12,6 +12,7 @@ import java.sql.SQLFeatureNotSupportedException; import java.sql.SQLType; import java.sql.Timestamp; +import java.sql.Types; import java.time.Duration; import java.time.LocalDateTime; import java.time.Period; @@ -176,4 +177,22 @@ static EsType of(Class clazz) throws SQLException { } return dataType; } + + static EsType baseType(EsType type) throws SQLException { + String typeName = type.getName(); + return isArray(type) + ? of(typeName.substring(0, typeName.length() - "_ARRAY".length()).toLowerCase(Locale.ROOT)) + : type; + } + + static EsType arrayOf(EsType type) throws SQLException { + if (isArray(type)) { + throw new SQLException("multidimentional array types not supported"); + } + return ENUM_NAME_TO_TYPE.get(type.name().toLowerCase(Locale.ROOT) + "_array"); + } + + static boolean isArray(EsType type) { + return type.getVendorTypeNumber() == Types.ARRAY; + } } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcArrayTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcArrayTests.java new file mode 100644 index 0000000000000..35672ae79ee33 --- /dev/null +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcArrayTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.jdbc; + +import org.elasticsearch.test.ESTestCase; + +import java.sql.Array; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.baseType; + +public class JdbcArrayTests extends ESTestCase { + + static final List ARRAY_TYPES = Arrays.stream(EsType.values()).filter(TypeUtils::isArray).collect(Collectors.toList()); + + public void testMetaData() throws Exception { + for (EsType arrayType : ARRAY_TYPES) { + Array array = new JdbcArray(arrayType, emptyList()); + + assertEquals(baseType(arrayType).getVendorTypeNumber().intValue(), array.getBaseType()); + assertEquals(baseType(arrayType).getName(), array.getBaseTypeName()); + } + } + + public void testGetArray() throws SQLException { + List expected = randomList(1, 10, ESTestCase::randomLong); + Array array = new JdbcArray(EsType.LONG_ARRAY, expected); + + List actual = asList((Object[]) array.getArray()); + assertEquals(expected, actual); + } + + public void testArraySlicing() throws SQLException { + List values = IntStream.rangeClosed(0, 9).boxed().collect(Collectors.toList()); + Array array = new JdbcArray(EsType.INTEGER_ARRAY, values); + + Object[] empty = (Object[]) array.getArray(11, 2); + assertEquals(0, empty.length); + + Object[] edgeSingleton = (Object[]) array.getArray(10, 2); + assertEquals(9, edgeSingleton[0]); + + Object[] midSingleton = (Object[]) array.getArray(5, 1); + assertEquals(4, midSingleton[0]); + + Object[] contained = (Object[]) array.getArray(4, 3); + assertEquals(asList(3, 4, 5), asList(contained)); + + Object[] overlapping = (Object[]) array.getArray(9, 3); + assertEquals(asList(8, 9), asList(overlapping)); + + SQLException sqle = expectThrows(SQLException.class, () -> array.getArray(0, 9)); + assertEquals("Index value [0] out of range [1, 2147483647]", sqle.getMessage()); + + sqle = expectThrows(SQLException.class, () -> array.getArray(Integer.MAX_VALUE + 1L, 9)); + assertEquals("Index value [2147483648] out of range [1, 2147483647]", sqle.getMessage()); + } +} diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java index 440c167cfcc35..e7e6094a6ded7 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcPreparedStatementTests.java @@ -21,6 +21,7 @@ import java.time.LocalDateTime; import java.time.ZonedDateTime; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.Locale; import java.util.Map; @@ -585,6 +586,13 @@ public void testThrownExceptionsWhenSettingByteArrayValuesToEsTypes() throws SQL assertEquals("Conversion from type [byte[]] to [INTERVAL_DAY_TO_MINUTE] not supported", sqle.getMessage()); } + public void testThrownExceptionWhenSettingArrayValue() throws SQLException { + JdbcPreparedStatement jps = createJdbcPreparedStatement(); + JdbcArray array = new JdbcArray(LONG, Collections.emptyList()); + SQLException sqle = expectThrows(SQLFeatureNotSupportedException.class, () -> jps.setArray(1, array)); + assertEquals("Objects of type [java.sql.Array] are not supported", sqle.getMessage()); + } + private JdbcPreparedStatement createJdbcPreparedStatement() throws SQLException { return new JdbcPreparedStatement(null, JdbcConfiguration.create("jdbc:es://l:1", null, 0), "?"); } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java index 999ffa6846bb7..26eaaedf89919 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java @@ -14,10 +14,20 @@ import java.sql.Date; import java.sql.Timestamp; +import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import static java.util.Arrays.asList; import static org.elasticsearch.xpack.sql.jdbc.JdbcTestUtils.nowWithMillisResolution; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.arrayOf; +import static org.elasticsearch.xpack.sql.jdbc.TypeUtils.of; import static org.hamcrest.Matchers.instanceOf; @@ -62,6 +72,42 @@ public void testDateAsNative() throws Exception { assertEquals(now.toLocalDate().atStartOfDay(ZoneId.of("Etc/GMT-10")).toInstant().toEpochMilli(), ((Date) nativeObject).getTime()); } + public void testMultiValueConvert() throws Exception { + Supplier supplier = randomFrom( + ESTestCase::randomBoolean, + ESTestCase::randomLong, + ESTestCase::randomDouble, + () -> ESTestCase.randomAlphaOfLengthBetween(1, 10), + () -> Timestamp.from(Instant.now()), + () -> null); + List expected = randomList(1, 10, supplier); + + Function toJsoned = x -> { + if (x instanceof Timestamp) { + String str = x.toString().replace(' ', 'T'); + int offset = TimeZone.getDefault().getRawOffset() / 1000; + str += String.format(Locale.ROOT, "%+03d:%02d", offset / 3600, (offset % 3600) / 60); + return str; + } else if (x instanceof Boolean || x instanceof Long || x == null) { + return x; + } else { + return x.toString(); + } + }; + List asFromJson = expected.stream().map(toJsoned).collect(Collectors.toList()); + EsType arrayType = expected.get(0) != null + ? arrayOf(of(expected.get(0).getClass())) + : randomFrom(EsType.BOOLEAN_ARRAY, EsType.LONG_ARRAY, EsType.DOUBLE_ARRAY, EsType.KEYWORD_ARRAY, EsType.DATETIME_ARRAY); + + assertNotNull(arrayType); + assertEquals(expected, TypeConverter.convert(asFromJson, arrayType, arrayType.getName())); + } + + public void testMultiValueFailedConversion() throws Exception { + expectThrows(ClassCastException.class, () -> convertAsNative(3L, EsType.LONG_ARRAY)); + expectThrows(ClassCastException.class, () -> convertAsNative(asList(3L), EsType.LONG)); + } + private Object convertAsNative(Object value, EsType type) throws Exception { // Simulate sending over XContent XContentBuilder builder = JsonXContent.contentBuilder(); diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java index 7d11bfb113bad..506b747c96e77 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcTestUtils.java @@ -17,8 +17,12 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Calendar; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Supplier; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; +import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.elasticsearch.test.ESTestCase.randomLongBetween; final class JdbcTestUtils { @@ -38,11 +42,13 @@ private JdbcTestUtils() {} * Until the feature + QA tests are actually ported to the target branch, the comparison will hold true only for the master * branch. So you'll need to remove the equality, port the feature and subsequently add the equality; i.e. a two-step commit of a PR. * - * public static boolean isUnsignedLongSupported() { - * // TODO: add equality only once actually ported to 7.11 - * return V_7_11_0.compareTo(JDBC_DRIVER_VERSION) < 0; + * static boolean isUnsignedLongSupported() { + * // TODO: add equality (onOrAfter) only once actually ported to 7.11 + * return JDBC_DRIVER_VERSION.after(Version.V_7_11_0); * } * + * Alternatively, use head's version for development (like V_8_0_0) with the right test (onOrAfter) and update it after porting, both + * in target branch and master. */ static final Version JDBC_DRIVER_VERSION; @@ -104,7 +110,22 @@ static int extractNanosOnly(long nanos) { return (int) (nanos % 1_000_000_000); } + static Set randomSet(int minSetSize, int maxSetSize, Supplier valueConstructor) { + final int size = randomIntBetween(minSetSize, maxSetSize); + Set set = new HashSet<>(); + while (set.size() < size) { + set.add(valueConstructor.get()); + } + return set; + } + static boolean versionSupportsDateNanos() { return JDBC_DRIVER_VERSION.onOrAfter(Version.V_7_12_0); } + + static boolean versionSupportsArrayTypes() { + // TODO: add equality (onOrAfter) only once actually ported to 7.12 + return JDBC_DRIVER_VERSION.after(Version.V_7_13_0); + + } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java index 435d4fe61fbc8..cee1475f4532f 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetMetaDataTestCase.java @@ -15,6 +15,12 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.util.Arrays; +import java.util.Locale; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.JDBC_DRIVER_VERSION; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.versionSupportsArrayTypes; public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase { @@ -30,12 +36,7 @@ public abstract class ResultSetMetaDataTestCase extends JdbcIntegrationTestCase "test_date" }; public void testValidGetObjectCalls() throws IOException, SQLException { - ResultSetTestCase.createIndex("test"); - ResultSetTestCase.updateMapping("test", builder -> { - for (String field : fieldsNames) { - builder.startObject(field).field("type", field.substring(5)).endObject(); - } - }); + setupTest(); String q = "SELECT test_byte, test_integer, test_long, test_short, test_double, test_float, test_keyword, " + "test_boolean, test_date FROM test"; @@ -46,6 +47,36 @@ public void testValidGetObjectCalls() throws IOException, SQLException { doWithQuery(q, r -> assertColumnNamesAndLabels(r.getMetaData(), new String[] { "b", "i", "l", "s", "d", "f", "k", "bool", "dt" })); } + public void testValidArrayTypes() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support array types", versionSupportsArrayTypes()); + + setupTest(); + + String q = "SELECT " + + Arrays.stream(fieldsNames).map(x -> "ARRAY(" + x + ")").collect(Collectors.joining(", ")) + + " FROM test"; + + doWithQuery(q, r -> { + ResultSetMetaData md = r.getMetaData(); + assertEquals(fieldsNames.length, md.getColumnCount()); + for (int i = 0; i < fieldsNames.length; i++) { + String typeName = fieldsNames[i].substring("test_".length()).toUpperCase(Locale.ROOT); + typeName = typeName.equals("DATE") ? "DATETIME" : typeName; + typeName += "_ARRAY"; + assertEquals(typeName, md.getColumnTypeName(i + 1)); + } + }); + } + + private void setupTest() throws IOException { + ResultSetTestCase.createIndex("test"); + ResultSetTestCase.updateMapping("test", builder -> { + for (String field : fieldsNames) { + builder.startObject(field).field("type", field.substring(5)).endObject(); + } + }); + } + private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { try (Connection connection = esJdbc()) { try (PreparedStatement statement = connection.prepareStatement(query)) { diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index 3567611f18f00..fe87c901c3d82 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; +import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.Connection; @@ -44,6 +45,7 @@ import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Calendar; +import java.util.Comparator; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; @@ -61,6 +63,7 @@ import java.util.stream.Stream; import static java.lang.String.format; +import static java.util.Arrays.asList; import static java.util.Calendar.DAY_OF_MONTH; import static java.util.Calendar.ERA; import static java.util.Calendar.HOUR_OF_DAY; @@ -78,7 +81,9 @@ import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.asTime; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.extractNanosOnly; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.of; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.randomSet; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.randomTimeInNanos; +import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.versionSupportsArrayTypes; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.versionSupportsDateNanos; import static org.hamcrest.Matchers.matchesPattern; @@ -135,7 +140,7 @@ public void testMultiValueFieldWithMultiValueLeniencyDisabled() throws IOExcepti SQLException.class, () -> doWithQuery(() -> esWithLeniency(false), "SELECT int, keyword FROM test", results -> {}) ); - assertTrue(expected.getMessage().contains("Arrays (returned by [int]) are not supported")); + assertTrue(expected.getMessage().contains("Cannot return multiple values for field [int]; use ARRAY(int) instead")); // default has multi value disabled expectThrows(SQLException.class, () -> doWithQuery(this::esJdbc, "SELECT int, keyword FROM test", results -> {})); @@ -171,7 +176,8 @@ public void testMultiValueFields_InsideObjects_WithMultiValueLeniencyDisabled() results -> {} ) ); - assertTrue(expected.getMessage().contains("Arrays (returned by [object.intsubfield]) are not supported")); + assertTrue(expected.getMessage().contains("Cannot return multiple values for field [object.intsubfield]; " + + "use ARRAY(object.intsubfield) instead")); // default has multi value disabled expectThrows( @@ -180,6 +186,49 @@ public void testMultiValueFields_InsideObjects_WithMultiValueLeniencyDisabled() ); } + public void testMultiValueFieldsAsArray() throws IOException, SQLException { + assumeTrue("Driver version [" + JDBC_DRIVER_VERSION + "] doesn't support array types", versionSupportsArrayTypes()); + + Integer[] ints = createTestDataForMultiValueTests(); + List strings = Arrays.stream(ints).map(String::valueOf).collect(Collectors.toList()); + + index("test", "2", builder -> { + builder.array("int", (Object) null); + builder.array("keyword", (Object) null); + }); + + + doWithQuery("SELECT ARRAY(int), ARRAY(keyword) FROM test", r -> { + assertTrue(r.next()); + + Array intArray = r.getArray(1); + assertEquals(EsType.INTEGER.getVendorTypeNumber().intValue(), intArray.getBaseType()); + + List expected = asList(ints); + expected.sort(Integer::compare); + List actual = asList((Object[]) intArray.getArray()); + actual.sort(Comparator.comparingInt(x -> (Integer) x)); + assertEquals(expected, actual); + + Array strArray = r.getArray(2); + assertEquals(EsType.KEYWORD.getVendorTypeNumber().intValue(), strArray.getBaseType()); + + strings.sort(String::compareTo); + actual = asList((Object[]) strArray.getArray()); + actual.sort(Comparator.comparing(x -> (String) x)); + assertEquals(strings, actual); + + assertTrue(r.next()); + + intArray = r.getArray(1); + assertEquals(0, ((Object[]) intArray.getArray()).length); + strArray = r.getArray(2); + assertEquals(0, ((Object[]) strArray.getArray()).length); + + assertFalse(r.next()); + }); + } + // Byte values testing public void testGettingValidByteWithoutCasting() throws IOException, SQLException { List byteTestValues = createTestDataForNumericValueTests(ESTestCase::randomByte); @@ -1836,8 +1885,6 @@ public void testUnsupportedGetMethods() throws IOException, SQLException { r.next(); assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getAsciiStream("test"), "AsciiStream not supported"); assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getAsciiStream(1), "AsciiStream not supported"); - assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getArray("test"), "Array not supported"); - assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getArray(1), "Array not supported"); assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getBinaryStream("test"), "BinaryStream not supported"); assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getBinaryStream(1), "BinaryStream not supported"); assertThrowsUnsupportedAndExpectErrorMessage(() -> r.getBlob("test"), "Blob not supported"); @@ -2071,14 +2118,15 @@ protected static void updateMapping(String index, CheckedConsumer { builder.startObject("int").field("type", "integer").endObject(); builder.startObject("keyword").field("type", "keyword").endObject(); }); - Integer[] values = randomArray(3, 15, Integer[]::new, () -> randomInt(50)); + // need a set to prevent duplicates in keyword multivalues being deduplicated by ES + Integer[] values = randomSet(3, 15, () -> randomInt(50)).toArray(new Integer[0]); // add the known value as the first one in list. Parsing from _source the value will pick up the first value in the array. values[0] = -10; @@ -2091,6 +2139,8 @@ private void createTestDataForMultiValueTests() throws IOException { builder.array("int", (Object[]) values); builder.array("keyword", stringValues); }); + + return values; } private void createTestDataForMultiValuesInObjectsTests() throws IOException { @@ -2139,7 +2189,7 @@ private void createTestDataForMultiValuesInObjectsTests() throws IOException { private List createTestDataForNumericValueTests(Supplier numberGenerator) throws IOException { T random1 = numberGenerator.get(); T random2 = randomValueOtherThan(random1, numberGenerator); - T random3 = randomValueOtherThanMany(Arrays.asList(random1, random2)::contains, numberGenerator); + T random3 = randomValueOtherThanMany(asList(random1, random2)::contains, numberGenerator); Class clazz = random1.getClass(); String primitiveName = clazz.getSimpleName().toLowerCase(Locale.ROOT); @@ -2160,7 +2210,7 @@ private List createTestDataForNumericValueTests(Supplier extractColumnTypesFromHeader(String header) } private static String resolveColumnType(String type) { - switch (type.toLowerCase(Locale.ROOT)) { + String t = type.toLowerCase(Locale.ROOT); + switch (t) { case "s": return "string"; case "b": @@ -163,7 +164,8 @@ private static String resolveColumnType(String type) { case "sh": return "short"; default: - return type; + // treat the arrays as strings; need for the CSV to correctly return the column value. + return t.endsWith("_a") || t.endsWith("_array") ? "string" : type; } } diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index 790762cb907d4..94090406ab4ec 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class DataLoader { @@ -73,6 +74,8 @@ protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception { // frozen index loadEmpDatasetIntoEs(client, "frozen_emp", "employees"); freeze(client, "frozen_emp"); + // multivalue + loadEcommerceDatasetIntoEs(client, "ecommerce", "ecommerce"); } public static void loadDocsDatasetIntoEs(RestClient client) throws Exception { @@ -179,8 +182,7 @@ private static void loadEmpDatasetIntoEs(RestClient client, String index, String list.add(dep); }); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); + request = new Request("POST", "/" + index + "/_bulk?refresh=true"); StringBuilder bulk = new StringBuilder(); csvToLines(fileName, (titles, fields) -> { bulk.append("{\"index\":{}}\n"); @@ -267,8 +269,7 @@ protected static void loadLogsDatasetIntoEs(RestClient client, String index, Str request.setJsonEntity(Strings.toString(createIndex)); client.performRequest(request); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); + request = new Request("POST", "/" + index + "/_bulk?refresh=true"); StringBuilder bulk = new StringBuilder(); csvToLines(filename, (titles, fields) -> { bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); @@ -310,8 +311,7 @@ protected static void loadLogNanosDatasetIntoEs(RestClient client, String index, request.setJsonEntity(Strings.toString(createIndex)); client.performRequest(request); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); + request = new Request("POST", "/" + index + "/_bulk?refresh=true"); StringBuilder bulk = new StringBuilder(); csvToLines(filename, (titles, fields) -> { bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); @@ -354,8 +354,7 @@ protected static void loadLibDatasetIntoEs(RestClient client, String index) thro request.setJsonEntity(Strings.toString(createIndex)); client.performRequest(request); - request = new Request("POST", "/" + index + "/_bulk?refresh=wait_for"); - request.addParameter("refresh", "true"); + request = new Request("POST", "/" + index + "/_bulk?refresh=true"); StringBuilder bulk = new StringBuilder(); csvToLines("library", (titles, fields) -> { bulk.append("{\"index\":{\"_id\":\"" + fields.get(0) + "\"}}\n"); @@ -372,6 +371,60 @@ protected static void loadLibDatasetIntoEs(RestClient client, String index) thro Response response = client.performRequest(request); } + // TODO: functionalize index creation (settings+mapping) and bulk builder throughout class? + protected static void loadEcommerceDatasetIntoEs(RestClient client, String index, String fileName) throws Exception { + XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); + createIndex.startObject("settings"); + { + createIndex.field("number_of_shards", 1); + createIndex.field("number_of_replicas", 1); + } + createIndex.endObject(); + createIndex.startObject("mappings"); + { + createIndex.startObject("properties"); + { + createIndex.startObject("id").field("type", "integer").endObject(); + createIndex.startObject("date").field("type", "date").endObject(); + createString("product", createIndex); + createString("sku", createIndex); + createIndex.startObject("price").field("type", "integer").endObject(); + createIndex.startObject("tax").field("type", "double").endObject(); + createIndex.startObject("shipped").field("type", "date").endObject(); + } + createIndex.endObject(); + } + createIndex.endObject().endObject(); + + Request request = new Request("PUT", "/" + index); + request.setJsonEntity(Strings.toString(createIndex)); + client.performRequest(request); + + StringBuilder builder = new StringBuilder(); + csvToLines(fileName, (headers, values) -> { + builder.append("{\"index\":{\"_id\":" + values.get(0) + "}}\n"); + builder.append("{"); + for (int i = 0; i < headers.size(); i++) { + String value = values.get(i); + if (i == 1) { // date: [2020-...] + value = '"' + value + '"'; + } else if (i == 6) { // shipped: [2020-..,2020-...] or [null,2020-...] + String[] tokens = value.substring(1, value.length() - 1).split(","); + value = Arrays.stream(tokens).map(x -> x.equals("null") ? x : '"' + x + '"').collect(Collectors.joining(",")); + value = '[' + value + ']'; + } + builder.append('"').append(headers.get(i)).append('"').append(':').append(value); + builder.append(","); + } + builder.deleteCharAt(builder.length() - 1); // trailing comma + builder.append("}").append("\n"); + }); + + request = new Request("POST", "/" + index + "/_bulk?refresh=true"); + request.setJsonEntity(builder.toString()); + client.performRequest(request); // fails by exception + } + public static void makeAlias(RestClient client, String aliasName, String... indices) throws Exception { for (String index : indices) { client.performRequest(new Request("POST", "/" + index + "/_alias/" + aliasName)); @@ -396,15 +449,58 @@ private static void csvToLines(String name, CheckedBiConsumer, List if (titlesString == null) { throw new IllegalArgumentException("[" + location + "] must contain at least a title row"); } - List titles = Arrays.asList(titlesString.split(",")); + List titles = splitCsvLine(titlesString); String line; while ((line = reader.readLine()) != null) { - consumeLine.accept(titles, Arrays.asList(line.split(","))); + List values = splitCsvLine(line); + if (values.size() != titles.size()) { + throw new IllegalArgumentException( + "Values count mismatch in header [" + titles.size() + "] and line " + "[" + values.size() + "] for [" + line + "]" + ); + } + consumeLine.accept(titles, values); } } } + private static List splitCsvLine(String line) { + List values = new ArrayList<>(); + StringBuilder buff = new StringBuilder(); + boolean quoteOn = false; + char crr, prev = 0; + for (int i = 0; i < line.length(); i++, prev = crr) { + switch ((crr = line.charAt(i))) { + case ',': + if (quoteOn) { + buff.append(crr); + } else { + values.add(buff.toString()); + buff = new StringBuilder(); + } + break; + case '"': + if (quoteOn) { + quoteOn = false; + } else { + if (prev == '"') { // double `"` == escaped `"` + buff.append('"'); + } // else: no if (prev != 0 && prev != ',') check => ...,foo"bar"baz,... -> foobarbaz + quoteOn = true; + } + break; + default: + buff.append(crr); + break; + } + } + if (quoteOn) { + throw new IllegalArgumentException("Quote not closed in badly formatted CSV line: [" + line + "]"); + } + values.add(buff.toString()); + return values; + } + @SuppressForbidden(reason = "test reads from jar") public static InputStream readFromJarUrl(URL source) throws IOException { return source.openStream(); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java index 4ab10119025af..fe3940d381b2e 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.sql.qa.jdbc; import com.carrotsearch.hppc.IntObjectHashMap; - import org.apache.logging.log4j.Logger; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; @@ -23,24 +22,39 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Timestamp; import java.sql.Types; import java.text.ParseException; +import java.time.ZonedDateTime; import java.time.temporal.TemporalAmount; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.TimeZone; +import java.util.function.Function; import static java.lang.String.format; +import static java.sql.Types.ARRAY; import static java.sql.Types.BIGINT; +import static java.sql.Types.BINARY; +import static java.sql.Types.BOOLEAN; +import static java.sql.Types.DATE; import static java.sql.Types.DOUBLE; import static java.sql.Types.FLOAT; import static java.sql.Types.INTEGER; +import static java.sql.Types.NULL; +import static java.sql.Types.OTHER; import static java.sql.Types.REAL; import static java.sql.Types.SMALLINT; +import static java.sql.Types.TIMESTAMP; +import static java.sql.Types.TIMESTAMP_WITH_TIMEZONE; import static java.sql.Types.TINYINT; +import static java.sql.Types.VARCHAR; import static java.time.ZoneOffset.UTC; +import static org.elasticsearch.xpack.sql.proto.StringUtils.ISO_DATETIME_WITH_NANOS; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.logResultSetMetaData; import static org.elasticsearch.xpack.sql.qa.jdbc.JdbcTestUtils.resultSetCurrentData; import static org.hamcrest.MatcherAssert.assertThat; @@ -60,6 +74,22 @@ public class JdbcAssert { private static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true)); + private static final Map> CSV_ARRAY_VALUES_CONVERTER_MAP = new HashMap<>() { + { + put(BOOLEAN, Boolean::valueOf); + put(TINYINT, Byte::valueOf); + put(SMALLINT, Short::valueOf); + put(INTEGER, Integer::valueOf); + put(BIGINT, Long::valueOf); + put(REAL, Float::valueOf); + put(FLOAT, Float::valueOf); + put(DOUBLE, Double::valueOf); + put(VARCHAR, x -> x.substring(1, x.length() - 1)); // strip framing quotes + put(BINARY, x -> x.substring(1, x.length() - 1)); + put(TIMESTAMP, JdbcAssert::asTimestamp); + } + }; + static { for (EsType type : EsType.values()) { SQL_TO_TYPE.putIfAbsent(type.getVendorTypeNumber().intValue(), type); @@ -167,26 +197,31 @@ public static void assertResultSetMetaData(ResultSet expected, ResultSet actual, // since H2 cannot use a fixed timezone, the data is stored in UTC (and thus with timezone) if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) { - expectedType = Types.TIMESTAMP; + expectedType = TIMESTAMP; } // H2 treats GEOMETRY as OTHER - if (expectedType == Types.OTHER && nameOf(actualType).startsWith("GEO_")) { - actualType = Types.OTHER; + if (expectedType == OTHER && nameOf(actualType).startsWith("GEO_")) { + actualType = OTHER; } // since csv doesn't support real, we use float instead..... - if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) { - expectedType = Types.REAL; + if (expectedType == FLOAT && expected instanceof CsvResultSet) { + expectedType = REAL; } // handle intervals - if ((expectedType == Types.VARCHAR && expected instanceof CsvResultSet) && nameOf(actualType).startsWith("INTERVAL_")) { + if ((expectedType == VARCHAR && expected instanceof CsvResultSet) && nameOf(actualType).startsWith("INTERVAL_")) { expectedType = actualType; } // csv doesn't support NULL type so skip type checking - if (actualType == Types.NULL && expected instanceof CsvResultSet) { - expectedType = Types.NULL; + if (actualType == NULL && expected instanceof CsvResultSet) { + expectedType = NULL; + } + + // csv doesn't support arrays + if (actualType == ARRAY) { + expectedType = ARRAY; } // when lenient is used, an int is equivalent to a short, etc... @@ -245,6 +280,8 @@ private static void doAssertResultSetData( for (int column = 1; column <= columns; column++) { int type = metaData.getColumnType(column); + int actualType = actual.getMetaData().getColumnType(column); + String actualTypeName = actual.getMetaData().getColumnTypeName(column); Class expectedColumnClass = null; try { String columnClassName = metaData.getColumnClassName(column); @@ -277,11 +314,6 @@ private static void doAssertResultSetData( throw new SQLException(cnfe); } - Object expectedObject = expected.getObject(column); - Object actualObject = (lenientDataType && expectedColumnClass != null) - ? actual.getObject(column, expectedColumnClass) - : actual.getObject(column); - String msg = format( Locale.ROOT, "Different result for column [%s], entry [%d]", @@ -289,54 +321,21 @@ private static void doAssertResultSetData( count + 1 ); - // handle nulls first - if (expectedObject == null || actualObject == null) { - // hack for JDBC CSV nulls - if (expectedObject != null && "null".equals(expectedObject.toString().toLowerCase(Locale.ROOT))) { - assertNull(msg, actualObject); - } else { - assertEquals(msg, expectedObject, actualObject); - } - } - // then timestamp - else if (type == Types.TIMESTAMP || type == Types.TIMESTAMP_WITH_TIMEZONE) { - assertEquals(msg, expected.getTimestamp(column), actual.getTimestamp(column)); - } - // then date - else if (type == Types.DATE) { - assertEquals(msg, convertDateToSystemTimezone(expected.getDate(column)), actual.getDate(column)); - } - // and floats/doubles - else if (type == Types.DOUBLE) { - assertEquals(msg, (double) expectedObject, (double) actualObject, lenientFloatingNumbers ? 1d : 0.0d); - } else if (type == Types.FLOAT) { - assertEquals(msg, (float) expectedObject, (float) actualObject, lenientFloatingNumbers ? 1f : 0.0f); - } else if (type == Types.OTHER) { - if (actualObject instanceof Geometry) { - // We need to convert the expected object to libs/geo Geometry for comparision - try { - expectedObject = WKT.fromWKT(expectedObject.toString()); - } catch (IOException | ParseException ex) { - fail(ex.getMessage()); - } - } - if (actualObject instanceof Point) { - // geo points are loaded form doc values where they are stored as long-encoded values leading - // to lose in precision - assertThat(expectedObject, instanceOf(Point.class)); - assertEquals(((Point) expectedObject).getY(), ((Point) actualObject).getY(), 0.000001d); - assertEquals(((Point) expectedObject).getX(), ((Point) actualObject).getX(), 0.000001d); - } else { - assertEquals(msg, expectedObject, actualObject); + Object expectedObject = extractObject(expected, column, false, null); + Object actualObject = extractObject(actual, column, true, lenientDataType ? expectedColumnClass : null); + if (actualType == ARRAY) { + int baseType = baseTypeOf(actualTypeName).getVendorTypeNumber(); + assertTrue(expectedObject instanceof String); + List expectedList = parseAsList((String) expectedObject, baseType); + + assertTrue(actualObject instanceof List); + List actualList = (List) actualObject; + assertEquals(expectedList.size(), actualList.size()); + for (int i = 0; i < actualList.size(); i++) { + compareValues(type, expectedList.get(i), actualList.get(i), lenientFloatingNumbers, msg); } - } - // intervals - else if (type == Types.VARCHAR && actualObject instanceof TemporalAmount) { - assertEquals(msg, expectedObject, StringUtils.toString(actualObject)); - } - // finally the actual comparison - else { - assertEquals(msg, expectedObject, actualObject); + } else { + compareValues(type, expectedObject, actualObject, lenientFloatingNumbers, msg); } } } @@ -353,6 +352,87 @@ else if (type == Types.VARCHAR && actualObject instanceof TemporalAmount) { } } + private static Object extractObject(ResultSet resultSet, int column, boolean fromEs, Class expectedColumnClass) throws SQLException { + switch (resultSet.getMetaData().getColumnType(column)) { + case TIMESTAMP: + case TIMESTAMP_WITH_TIMEZONE: + return resultSet.getTimestamp(column); + case DATE: + Date date = resultSet.getDate(column); + return fromEs == false && date != null ? convertDateToSystemTimezone(date) : date; + } + return expectedColumnClass == null ? resultSet.getObject(column) : resultSet.getObject(column, expectedColumnClass); + } + + private static void compareValues(int type, Object expectedObject, Object actualObject, boolean lenientFloatingNumbers, String msg) { + // handle nulls first + if (expectedObject == null || actualObject == null) { + // hack for JDBC CSV nulls + if (expectedObject != null && "null".equals(expectedObject.toString().toLowerCase(Locale.ROOT))) { + assertNull(msg, actualObject); + } else { + assertEquals(msg, expectedObject, actualObject); + } + } + // and floats/doubles + else if (type == DOUBLE) { + assertEquals(msg, (double) expectedObject, (double) actualObject, lenientFloatingNumbers ? 1d : 0.0d); + } else if (type == FLOAT) { + assertEquals(msg, (float) expectedObject, (float) actualObject, lenientFloatingNumbers ? 1f : 0.0f); + } else if (type == OTHER) { + if (actualObject instanceof Geometry) { + // We need to convert the expected object to libs/geo Geometry for comparision + try { + expectedObject = WKT.fromWKT(expectedObject.toString()); + } catch (IOException | ParseException ex) { + fail(ex.getMessage()); + } + } + if (actualObject instanceof Point) { + assertThat(expectedObject, instanceOf(Point.class)); + assertEquals(((Point) expectedObject).getY(), ((Point) actualObject).getY(), 0.000001d); + assertEquals(((Point) expectedObject).getX(), ((Point) actualObject).getX(), 0.000001d); + } else { + assertEquals(msg, expectedObject, actualObject); + } + } + // intervals + else if (type == VARCHAR && actualObject instanceof TemporalAmount) { + assertEquals(msg, expectedObject, StringUtils.toString(actualObject)); + } + // finally the actual comparison + else { + assertEquals(msg, expectedObject, actualObject); + } + + } + + private static List parseAsList(String string, int type) { + assertTrue("Not a list representation: [" + string + "]", string.startsWith("[") && string.endsWith("]")); + List list = new ArrayList<>(); + String unframed = string.substring(1, string.length() - 1); + if (unframed.isEmpty() == false) { + String[] tokens = unframed.split(","); + for (String tok : tokens) { + list.add(CSV_ARRAY_VALUES_CONVERTER_MAP.getOrDefault(type, x -> x).apply(tok)); + } + } + return list; + } + + private static Timestamp asTimestamp(String date) { + ZonedDateTime zdt = ISO_DATETIME_WITH_NANOS.parse(date, ZonedDateTime::from); + Timestamp timestamp = new Timestamp(zdt.toInstant().toEpochMilli()); + timestamp.setNanos(zdt.getNano()); + return timestamp; + } + + private static EsType baseTypeOf(String arrayTypeName) { + String typeName = arrayTypeName.toUpperCase(Locale.ROOT); + assertTrue(typeName.endsWith("_ARRAY")); + return EsType.valueOf(typeName.substring(0, typeName.length() - "_ARRAY".length())); + } + /** * Returns the value of the given type either in a lenient fashion (widened) or strict. */ diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index 5d9341e799377..2a126f3df5874 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -42,6 +42,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -733,7 +735,6 @@ public void testBasicQueryWithParameters() throws IOException { ); } - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testBasicQueryWithMultiValues() throws IOException { List values = randomList(1, 5, ESTestCase::randomLong); String field = randomAlphaOfLength(5); @@ -754,7 +755,6 @@ public void testBasicQueryWithMultiValues() throws IOException { ); } - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testBasicQueryWithMultiValuesAndMultiPathAndMultiDoc() throws IOException { // formatter will leave first argument as is, but fold the following on a line index( @@ -795,9 +795,9 @@ public void testBasicQueryWithMultiValuesAndMultiPathAndMultiDoc() throws IOExce Map expected = new HashMap<>(); expected.put("columns", singletonList(columnInfo(mode, "ARRAY(a.b.c.d)", "long_array", JDBCType.ARRAY, 20))); if (columnar) { - expected.put("values", singletonList(asList(asList(2, 3, 4, 5), singletonList(6), asList(8, 7)))); + expected.put("values", singletonList(asList(asList(5, 2, 3, 4), singletonList(6), asList(7, 8)))); } else { - expected.put("rows", asList(singletonList(asList(2, 3, 4, 5)), singletonList(singletonList(6)), singletonList(asList(8, 7)))); + expected.put("rows", asList(singletonList(asList(5, 2, 3, 4)), singletonList(singletonList(6)), singletonList(asList(7, 8)))); } assertResponse( @@ -813,7 +813,6 @@ public void testBasicQueryWithMultiValuesAndMultiPathAndMultiDoc() throws IOExce ); } - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testFilteringQueryWithMultiValuesAndWithout() throws IOException { index("{\"a\": [2, 3, 4, 5]}", "{\"a\": 6}", "{\"a\": [7, 8]}"); String mode = randomMode(); @@ -837,6 +836,29 @@ public void testFilteringQueryWithMultiValuesAndWithout() throws IOException { ); } + @SuppressWarnings("unchecked") + public void testMultiValueDifferentPathsWithArraysAndNulls() throws IOException { + Supplier supplier = () -> rarely() ? null : randomLong(); + ExhaustiveMultiPathMapper mapper = new ExhaustiveMultiPathMapper<>(randomIntBetween(3, 10), supplier); + + index(Strings.toString(JsonXContent.contentBuilder().map(mapper.map()))); + + Map retMap = runSql(randomMode(), "SELECT ARRAY(\\\"" + mapper.path() + "\\\") FROM test", false); + List rows = (List) retMap.get("rows"); + assertTrue(rows.size() == 1 && rows.get(0) instanceof List && ((List) rows.get(0)).size() == 1); + List cell = (List) ((List) rows.get(0)).get(0); + assertFalse(cell.isEmpty()); + assertTrue(cell.get(0) instanceof Long); + List floats = (List) cell; + + // order of array retrieval is stable, not trivial to predict and ultimately irrelevant / not guaranteed + floats.sort(Long::compare); + List values = mapper.values(); + values.removeIf(Objects::isNull); + values.sort(Long::compare); + assertEquals(values, floats); + } + public void testBasicTranslateQueryWithFilter() throws IOException { index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}"); @@ -1100,7 +1122,6 @@ private void executeQueryWithNextPage(String format, String expectedHeader, Stri assertEquals(0, getNumberOfSearchContexts(client(), "test")); } - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testMultiValueQueryText() throws IOException { index( "{" @@ -1118,7 +1139,7 @@ public void testMultiValueQueryText() throws IOException { String expected = " t | n \n" + "-------------------------------+---------------\n" - + "[\"one\",\"two, three\",\"\\\"four\\\"\"]|[1,2,3,4] \n"; + + "[\"one\",\"two, three\",\"\\\"four\\\"\"]|[1,4,2,3] \n"; Tuple response = runSqlAsText("SELECT ARRAY(text) t, ARRAY(number) n FROM test", "text/plain"); assertEquals(expected, response.v1()); } @@ -1169,4 +1190,171 @@ public static void assertResponse(Map expected, Map { + private final List> paths; + private final String path; + private Map map; + private List values; + + // the depth excludes the leaf + ExhaustiveMultiPathMapper(int depth, Supplier supplier) { + if (depth < 2) { + throw new IllegalArgumentException("the depth must be larger than 2; provided: [" + depth + "]"); + } + // generate the list of nodes ("a", "b", "c"...) + List nodes = randomList(2, depth, () -> randomAlphaOfLength(randomIntBetween(1, 5))); + paths = generatePaths(nodes); + values = new ArrayList<>(paths.size()); + path = String.join(".", paths.get(0)); + + generate(supplier); + } + + Map map() { + return map; + } + + List values() { + return values; + } + + String path() { + return path; + } + + @SuppressWarnings("unchecked") + private void generate(Supplier supplier) { + values = new ArrayList<>(paths.size()); + map = new HashMap<>(); + + // add the leaf values to all paths + for (List path : paths) { + Object value; + // randomly generate single value or a list of them + if (randomBoolean()) { + value = supplier.get(); + values.add((T) value); + } else { + value = randomList(1, 5, supplier); + values.addAll((List) value); + } + if (path.size() == 1) { // "a.b.c": 3 + map.put(path.get(0), value); + } else { + // for a path ["a", "b", "c"] construct the JSON-like map of a path {"a": {"b": {"c": ...}}} + Map crrMap = new HashMap<>(); + crrMap.put(path.get(path.size() - 1), value); + for (int j = path.size() - 2; j > 0; j--) { + Map newMap = new HashMap<>(); + newMap.put(path.get(j), crrMap); + crrMap = newMap; + } + mergeMaps(map, path.get(0), crrMap); + } + } + randomlyMultiplySubmaps(map, values); + } + + // "a": {"b": 2} => "a": [{"b": 2}, {"b": 2}] + private static void randomlyMultiplySubmaps(Map map, List valuesList) { + map.keySet().forEach(key -> { + Object val = map.get(key); + if (val instanceof Map) { + @SuppressWarnings("unchecked") + Map innerMap = (Map) val; + if (usually()) { + List> replacementList = new ArrayList<>(); + int multiplicate = randomIntBetween(1, 5); + for (int i = 0; i < multiplicate; i++) { + Map copy = new HashMap<>(innerMap); + replacementList.add(copy); + if (i > 0) { // the initial copy is already part of valuesList + collectLeaves(copy, valuesList); + } + } + map.put(key, replacementList); + } else { + randomlyMultiplySubmaps(innerMap, valuesList); + } + } + }); + } + + @SuppressWarnings("unchecked") + private static void collectLeaves(Map map, List valuesList) { + for (Object val : map.values()) { + if (val instanceof Map) { + collectLeaves((Map) val, valuesList); + } else if (val instanceof List) { + for (Object o : (List) val) { + if (o instanceof Map) { + collectLeaves((Map) o, valuesList); + } else { + valuesList.add((T) o); + } + } + } else { + valuesList.add((T) val); + } + } + } + + // {"a" : {"b": {"c": 3}}} + "a", {"b.c": 4} => {"a": {"b": {"c": 3}, "b.c": 4}} + @SuppressWarnings("unchecked") + private static void mergeMaps(Map destination, String key, Map singleKeys) { + Object o = singleKeys; + while (destination.containsKey(key)) { + destination = (Map) destination.get(key); + key = singleKeys.keySet().toArray(new String[0])[0]; + o = singleKeys.get(key); + if (o instanceof Map == false) { + break; + } + singleKeys = (Map) o; + } + destination.put(key, o); + } + + // generate all possible path combinations with given node names: a, b, c => (a, b, c), (a, b.c), (a.b, c), (a.b.c) + private static List> generatePaths(List nodes) { + if (nodes.size() == 0) { + return emptyList(); + } + List> paths = new ArrayList<>(singletonList(singletonList(nodes.get(0)))); + for (int i = 1; i < nodes.size(); i++) { + List> newPaths = new ArrayList<>(); + for (List crrPath : paths) { + newPaths.addAll(extendPaths(crrPath, nodes.get(i))); + } + paths = newPaths; + } + return paths; + } + + // (a, b) + c => (a, b, c), (a, bc) + private static List> extendPaths(List paths, String node) { + List> extendedPaths = new ArrayList<>(paths.size() * 2); + List listA = new ArrayList<>(paths); + listA.add(node); + extendedPaths.add(listA); + if (paths.isEmpty() == false) { + List listB; + if (paths.size() > 1) { + listB = paths.subList(0, paths.size() - 1); + listB.add(paths.get(paths.size() - 1) + "." + node); + } else { + listB = singletonList(paths.get(0) + "." + node); + } + extendedPaths.add(listB); + } + return extendedPaths; + } + } } diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index 393a159bcc4f1..2658919afc59a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -225,7 +225,8 @@ TODAY |SCALAR showTables SHOW TABLES; - name | type | kind + name | type | kind +ecommerce |TABLE |INDEX logs |TABLE |INDEX logs_nanos |TABLE |INDEX test_alias |VIEW |ALIAS diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/ecommerce.csv b/x-pack/plugin/sql/qa/server/src/main/resources/ecommerce.csv new file mode 100644 index 0000000000000..80b5f11496a68 --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/ecommerce.csv @@ -0,0 +1,101 @@ +id,date,product,sku,price,tax,shipped +550375,2016-12-01T00:04:19.000Z,"[""Basic T-shirt - Medium Slate Blue"",""Tracksuit bottoms - black""]","[""ZO0567505675"",""ZO0622006220""]","[11,25]","[2.09,4.75]","[2016-12-03T00:04:19.000Z,2016-12-03T10:04:19.000Z]" +550385,2016-12-01T00:08:38.000Z,"[""Bomber Jacket - gold"",""Rucksack - brandy""]","[""ZO0267102671"",""ZO0357703577""]","[50,33]","[9.5,6.27]","[2016-12-03T00:08:38.000Z,2016-12-03T10:08:38.000Z]" +550405,2016-12-01T00:23:02.000Z,"[""Trainers - rosa"",""Scarf - offwhite""]","[""ZO0083400834"",""ZO0240802408""]","[60,12]","[11.4,2.28]","[2016-12-03T00:23:02.000Z,2016-12-03T10:23:02.000Z]" +550412,2016-12-01T00:33:07.000Z,"[""Trainers - white/black"",""Lace-ups - black ""]","[""ZO0508905089"",""ZO0681206812""]","[21,60]","[3.99,11.4]","[2016-12-03T00:33:07.000Z,2016-12-03T10:33:07.000Z]" +550425,2016-12-01T00:47:31.000Z,"[""Boots - black"",""Trainers - navy""]","[""ZO0256602566"",""ZO0516305163""]","[65,21]","[12.35,3.99]","[2016-12-03T00:47:31.000Z,2016-12-03T10:47:31.000Z]" +728387,2016-12-01T00:50:24.000Z,"[""Across body bag - cognac"",""Sandals - black"",""Cocktail dress / Party dress - eclipse"",""Tote bag - brown""]","[""ZO0089900899"",""ZO0205902059"",""ZO0242802428"",""ZO0338503385""]","[21,50,75,25]","[3.99,9.5,14.25,4.75]","[2016-12-03T00:50:24.000Z,2016-12-03T10:50:24.000Z]" +550428,2016-12-01T00:54:43.000Z,"[""Wallet - red"",""Weekend bag - black""]","[""ZO0195801958"",""ZO0303703037""]","[22,42]","[4.18,7.98]","[2016-12-03T00:54:43.000Z,2016-12-03T10:54:43.000Z]" +550452,2016-12-01T01:16:19.000Z,"[""Boots - tan"",""Across body bag - black""]","[""ZO0325803258"",""ZO0696306963""]","[65,60]","[12.35,11.4]","[2016-12-03T01:16:19.000Z,2016-12-03T11:16:19.000Z]" +550466,2016-12-01T01:27:50.000Z,"[""Summer dress - grey"",""Boots - passion""]","[""ZO0260702607"",""ZO0363203632""]","[50,100]","[9.5,19]","[2016-12-03T01:27:50.000Z,2016-12-03T11:27:50.000Z]" +550471,2016-12-01T01:32:10.000Z,"[""Ankle boots - black"",""Ballet pumps - black""]","[""ZO0143501435"",""ZO0235902359""]","[29,50]","[5.51,9.5]","[2016-12-03T01:32:10.000Z,2016-12-03T11:32:10.000Z]" +550472,2016-12-01T01:37:55.000Z,"[""Boots - cognac"",""High heels - black""]","[""ZO0243802438"",""ZO0667606676""]","[115,65]","[21.85,12.35]","[2016-12-03T01:37:55.000Z,2016-12-03T11:37:55.000Z]" +550473,2016-12-01T01:39:22.000Z,"[""Lace-up boots - dark tan"",""Jumper - dark blue""]","[""ZO0450504505"",""ZO0688006880""]","[65,38]","[12.35,7.22]","[2016-12-03T01:39:22.000Z,2016-12-03T11:39:22.000Z]" +731007,2016-12-01T01:53:46.000Z,"[""Jumper - Pale Violet Red"",""Clutch - natural"",""Cardigan - black"",""Jersey dress - black""]","[""ZO0047800478"",""ZO0308303083"",""ZO0351903519"",""ZO0652506525""]","[33,25,20,29]","[6.27,4.75,3.8,5.51]","[2016-12-03T01:53:46.000Z,2016-12-03T11:53:46.000Z]" +550503,2016-12-01T02:15:22.000Z,"[""Tracksuit top - black"",""Print T-shirt - khaki""]","[""ZO0566405664"",""ZO0587505875""]","[34,12]","[6.46,2.28]",[null] +550519,2016-12-01T02:29:46.000Z,"[""Ankle boots - ice"",""Sandals - black""]","[""ZO0018400184"",""ZO0137001370""]","[25,33]","[4.75,6.27]","[2016-12-03T02:29:46.000Z,2016-12-03T12:29:46.000Z]" +550521,2016-12-01T02:29:46.000Z,"[""Basic T-shirt - light blue"",""Classic heels - black""]","[""ZO0666706667"",""ZO0710307103""]","[12,65]","[2.28,12.35]","[2016-12-03T02:29:46.000Z,2016-12-03T12:29:46.000Z]" +550522,2016-12-01T02:31:12.000Z,"[""Sandals - black"",""3 PACK - Socks - blue/grey""]","[""ZO0258502585"",""ZO0481004810""]","[33,8]","[6.27,1.52]","[2016-12-03T02:31:12.000Z,2016-12-03T12:31:12.000Z]" +550538,2016-12-01T02:51:22.000Z,"[""Handbag - black"",""Ankle boots - grey""]","[""ZO0246202462"",""ZO0699406994""]","[75,60]","[14.25,11.4]","[2016-12-03T02:51:22.000Z,2016-12-03T12:51:22.000Z]" +550542,2016-12-01T02:54:14.000Z,"[""High heeled sandals - cognac"",""Snood - black/red/green/yellow""]","[""ZO0137301373"",""ZO0192601926""]","[25,10]","[4.75,1.9]","[2016-12-03T02:54:14.000Z,2016-12-03T12:54:14.000Z]" +550568,2016-12-01T03:25:55.000Z,"[""Casual lace-ups - navy"",""Jumper - dark grey multicolor""]","[""ZO0388403884"",""ZO0447604476""]","[50,25]","[9.5,4.75]",[2016-12-03T13:25:55.000Z] +550569,2016-12-01T03:28:48.000Z,"[""Lace-ups - camel/saphire"",""Lace-up boots - taupe""]","[""ZO0389303893"",""ZO0688606886""]","[50,65]","[9.5,12.35]",[2016-12-03T13:28:48.000Z] +550578,2016-12-01T03:38:53.000Z,"[""Lace-up boots - cognac/whisky"",""Print T-shirt - rose dotted""]","[""ZO0326203262"",""ZO0707207072""]","[85,12]","[16.15,2.28]","[null,2016-12-03T13:38:53.000Z]" +550580,2016-12-01T03:41:46.000Z,"[""Lace-up boots - cognac"",""Sports shirt - black""]","[""ZO0144801448"",""ZO0219602196""]","[50,17]","[9.5,3.23]","[2016-12-03T03:41:46.000Z,2016-12-03T13:41:46.000Z]" +550588,2016-12-01T03:51:50.000Z,"[""Chinos - tan"",""Watch - black""]","[""ZO0528205282"",""ZO0601406014""]","[21,21]","[3.99,3.99]","[2016-12-03T03:51:50.000Z,2016-12-03T13:51:50.000Z]" +550609,2016-12-01T04:06:14.000Z,"[""2 PACK - Vest - white/white"",""Boots - black""]","[""ZO0139601396"",""ZO0642106421""]","[15,42]","[2.85,7.98]",[2016-12-03T04:06:14.000Z] +550622,2016-12-01T04:16:19.000Z,"[""Ankle boots - White"",""High heels - black""]","[""ZO0133201332"",""ZO0248802488""]","[75,26]","[14.25,4.94]","[2016-12-03T04:16:19.000Z,2016-12-03T14:16:19.000Z]" +550644,2016-12-01T04:42:14.000Z,"[""Blazer - brown"",""Lace-ups - black""]","[""ZO0270102701"",""ZO0368603686""]","[50,55]","[9.5,10.45]","[2016-12-03T04:42:14.000Z,2016-12-03T14:42:14.000Z]" +550653,2016-12-01T04:53:46.000Z,"[""SET - Pyjamas - grey/blue"",""Wool - black""]","[""ZO0478204782"",""ZO0627106271""]","[25,33]","[4.75,6.27]","[2016-12-03T04:53:46.000Z,2016-12-03T14:53:46.000Z]" +550659,2016-12-01T05:00:58.000Z,"[""Suit jacket - light grey"",""Sports shirt - dark grey""]","[""ZO0424004240"",""ZO0614706147""]","[60,11]","[11.4,2.09]","[null,null]" +550663,2016-12-01T05:05:17.000Z,"[""Lace-up boots - black"",""Print T-shirt - white""]","[""ZO0399703997"",""ZO0560905609""]","[65,11]","[12.35,2.09]","[2016-12-03T05:05:17.000Z,2016-12-03T15:05:17.000Z]" +550670,2016-12-01T05:16:48.000Z,"[""Ankle boots - cognac"",""Summer dress - Lemon Chiffon""]","[""ZO0143001430"",""ZO0336403364""]","[33,55]","[6.27,10.45]","[2016-12-03T05:16:48.000Z,2016-12-03T15:16:48.000Z]" +550687,2016-12-01T05:35:31.000Z,"[""Across body bag - gun "",""Jumper - dark rose""]","[""ZO0211202112"",""ZO0708607086""]","[19,22]","[3.61,4.18]","[2016-12-03T05:35:31.000Z,2016-12-03T15:35:31.000Z]" +550694,2016-12-01T05:42:43.000Z,"[""Suit trousers - black"",""Sweatshirt - dark blue""]","[""ZO0522805228"",""ZO0583805838""]","[29,25]","[5.51,4.75]","[2016-12-03T05:42:43.000Z,null]" +550708,2016-12-01T05:57:07.000Z,"[""Sweatshirt - dark blue"",""Blouse - black""]","[""ZO0181501815"",""ZO0650306503""]","[25,19]","[4.75,3.61]","[2016-12-03T05:57:07.000Z,2016-12-03T15:57:07.000Z]" +550709,2016-12-01T06:00:00.000Z,"[""Clutch - gold"",""Ballet pumps - khaki""]","[""ZO0001500015"",""ZO0698306983""]","[50,25]","[9.5,4.75]",[2016-12-03T06:00:00.000Z] +550710,2016-12-01T06:00:00.000Z,"[""Print T-shirt - navy blazer"",""Summer jacket - grey""]","[""ZO0118901189"",""ZO0285702857""]","[15,42]","[2.85,7.98]","[2016-12-03T06:00:00.000Z,null]" +550718,2016-12-01T06:07:12.000Z,"[""Jersey dress - white"",""Vest - salt/pepper""]","[""ZO0051100511"",""ZO0346903469""]","[33,33]","[6.27,6.27]","[2016-12-03T06:07:12.000Z,2016-12-03T16:07:12.000Z]" +550724,2016-12-01T06:14:24.000Z,"[""Long sleeved top - dark red"",""Platform boots - brown""]","[""ZO0145101451"",""ZO0163001630""]","[17,42]","[3.23,7.98]","[2016-12-03T06:14:24.000Z,2016-12-03T16:14:24.000Z]" +550728,2016-12-01T06:17:17.000Z,"[""Jeans Skinny Fit - khaki"",""Handbag - black""]","[""ZO0088500885"",""ZO0183701837""]","[25,31]","[4.75,5.89]",[2016-12-03T16:17:17.000Z] +550736,2016-12-01T06:27:22.000Z,"[""Dressing gown - off-white"",""Tote bag - taupe ""]","[""ZO0088200882"",""ZO0099600996""]","[21,33]","[3.99,6.27]","[null,2016-12-03T16:27:22.000Z]" +550743,2016-12-01T06:34:34.000Z,"[""Wallet - lilac"",""Over-the-knee boots - cognac""]","[""ZO0190401904"",""ZO0323803238""]","[12,100]","[2.28,19]","[2016-12-03T06:34:34.000Z,2016-12-03T16:34:34.000Z]" +550771,2016-12-01T07:09:07.000Z,"[""Blouse - Medium Slate Blue"",""Shift dress - Lemon Chiffon""]","[""ZO0265902659"",""ZO0332103321""]","[33,42]","[6.27,7.98]","[2016-12-03T07:09:07.000Z,2016-12-03T17:09:07.000Z]" +550784,2016-12-01T07:27:50.000Z,"[""Smart lace-ups - black"",""Lace-up boots - cognac""]","[""ZO0683206832"",""ZO0687706877""]","[75,65]","[14.25,12.35]","[null,null]" +550810,2016-12-01T07:56:38.000Z,"[""Summer jacket - black"",""Seratonin - Long sleeved top - khaki""]","[""ZO0538505385"",""ZO0547305473""]","[42,11]","[7.98,2.09]",[null] +550817,2016-12-01T08:03:50.000Z,"[""Shorts - black"",""Blazer - black""]","[""ZO0270302703"",""ZO0705107051""]","[8,65]","[1.52,12.35]","[2016-12-03T08:03:50.000Z,2016-12-03T18:03:50.000Z]" +550828,2016-12-01T08:21:07.000Z,"[""Tracksuit bottoms - black"",""Print T-shirt - off white""]","[""ZO0111201112"",""ZO0438704387""]","[29,14]","[5.51,2.66]","[2016-12-03T08:21:07.000Z,2016-12-03T18:21:07.000Z]" +550833,2016-12-01T08:24:00.000Z,"[""Ankle boots - sand/gold"",""Ankle boots - taupe""]","[""ZO0025700257"",""ZO0140901409""]","[42,29]","[7.98,5.51]","[null,null]" +550841,2016-12-01T08:32:38.000Z,"[""Jumper - multicolor"",""Snood - grey""]","[""ZO0449404494"",""ZO0603106031""]","[29,15]","[5.51,2.85]","[2016-12-03T08:32:38.000Z,2016-12-03T18:32:38.000Z]" +550861,2016-12-01T08:45:36.000Z,"[""Parka - dark denim"",""Shirt - light blue denim""]","[""ZO0111101111"",""ZO0626206262""]","[65,33]","[12.35,6.27]","[2016-12-03T08:45:36.000Z,2016-12-03T18:45:36.000Z]" +550864,2016-12-01T08:47:02.000Z,"[""Shorts - multicoloured"",""Shirt - grey/white""]","[""ZO0277802778"",""ZO0532905329""]","[21,25]","[3.99,4.75]","[2016-12-03T08:47:02.000Z,2016-12-03T18:47:02.000Z]" +550884,2016-12-01T09:01:26.000Z,"[""Clutch - silver"",""Snood - grey/lilac/black""]","[""ZO0096800968"",""ZO0191601916""]","[21,11]","[3.99,2.09]","[2016-12-03T09:01:26.000Z,2016-12-03T19:01:26.000Z]" +550892,2016-12-01T09:08:38.000Z,"[""Across body bag - black"",""Print T-shirt - white/black""]","[""ZO0464704647"",""ZO0558305583""]","[25,15]","[4.75,2.85]","[2016-12-03T09:08:38.000Z,2016-12-03T19:08:38.000Z]" +550896,2016-12-01T09:17:17.000Z,"[""Maxi skirt - blue"",""Lace-ups - white""]","[""ZO0633506335"",""ZO0669306693""]","[14,65]","[2.66,12.35]","[2016-12-03T09:17:17.000Z,2016-12-03T19:17:17.000Z]" +550900,2016-12-01T09:21:36.000Z,"[""Sandals - Misty Rose"",""Ankle boots - tan""]","[""ZO0242002420"",""ZO0246102461""]","[33,60]","[6.27,11.4]","[2016-12-03T09:21:36.000Z,2016-12-03T19:21:36.000Z]" +550921,2016-12-01T09:47:31.000Z,"[""Suit - navy"",""Basic T-shirt - blue""]","[""ZO0407604076"",""ZO0549905499""]","[110,12]","[20.9,2.28]",[2016-12-03T19:47:31.000Z] +550936,2016-12-01T10:03:22.000Z,"[""Blouse - pomegranate"",""Lace-up boots - black""]","[""ZO0065700657"",""ZO0379103791""]","[29,65]","[5.51,12.35]","[2016-12-03T10:03:22.000Z,2016-12-03T20:03:22.000Z]" +550942,2016-12-01T10:10:34.000Z,"[""Vest - gold metallic"",""Summer dress - blue""]","[""ZO0062400624"",""ZO0489404894""]","[19,29]","[3.61,5.51]","[2016-12-03T10:10:34.000Z,2016-12-03T20:10:34.000Z]" +550951,2016-12-01T10:17:46.000Z,"[""Cardigan - light grey"",""Wool jumper - dark blue""]","[""ZO0485504855"",""ZO0497504975""]","[33,21]","[6.27,3.99]","[2016-12-03T10:17:46.000Z,2016-12-03T20:17:46.000Z]" +550955,2016-12-01T10:20:38.000Z,"[""Print T-shirt - port royal"",""Blouse - dark red""]","[""ZO0062900629"",""ZO0651406514""]","[21,17]","[3.99,3.23]","[2016-12-03T10:20:38.000Z,2016-12-03T20:20:38.000Z]" +550957,2016-12-01T10:23:31.000Z,"[""Classic heels - petrol"",""Watch - nude""]","[""ZO0133101331"",""ZO0189401894""]","[25,17]","[4.75,3.23]","[2016-12-03T10:23:31.000Z,2016-12-03T20:23:31.000Z]" +723321,2016-12-01T10:33:36.000Z,"[""Slip-ons - grey"",""High heeled sandals - black"",""Across body bag - black"",""Pencil skirt - light brown""]","[""ZO0010900109"",""ZO0136201362"",""ZO0209602096"",""ZO0259902599""]","[25,29,14,33]","[4.75,5.51,2.66,6.27]","[2016-12-03T10:33:36.000Z,2016-12-03T20:33:36.000Z]" +550984,2016-12-01T10:55:12.000Z,"[""Sunglasses - black"",""Print T-shirt - light grey multicolor""]","[""ZO0548405484"",""ZO0600006000""]","[11,15]","[2.09,2.85]",[2016-12-03T10:55:12.000Z] +550989,2016-12-01T10:56:38.000Z,"[""Ankle boots - black"",""Summer dress - dusty rose""]","[""ZO0022200222"",""ZO0340503405""]","[33,50]","[6.27,9.5]","[2016-12-03T10:56:38.000Z,2016-12-03T20:56:38.000Z]" +550990,2016-12-01T10:56:38.000Z,"[""Lace-up boots - dark brown"",""Polo shirt - white""]","[""ZO0404404044"",""ZO0570605706""]","[50,12]","[9.5,2.28]","[2016-12-03T10:56:38.000Z,2016-12-03T20:56:38.000Z]" +551008,2016-12-01T11:15:22.000Z,"[""Print T-shirt - black"",""Across body bag - Blue Violety""]","[""ZO0207902079"",""ZO0491404914""]","[12,22]","[2.28,4.18]","[2016-12-03T11:15:22.000Z,2016-12-03T21:15:22.000Z]" +551018,2016-12-01T11:29:46.000Z,"[""Tote bag - gunmetal"",""Ankle boots - dark blue""]","[""ZO0198201982"",""ZO0377003770""]","[21,75]","[3.99,14.25]",[2016-12-03T21:29:46.000Z] +551026,2016-12-01T11:34:05.000Z,"[""Across body bag - black"",""Print T-shirt - Medium Sea Green""]","[""ZO0104501045"",""ZO0204202042""]","[17,23]","[3.23,4.37]","[2016-12-03T11:34:05.000Z,2016-12-03T21:34:05.000Z]" +551032,2016-12-01T11:41:17.000Z,"[""Lace-ups - gold"",""Blouse - black""]","[""ZO0059000590"",""ZO0669406694""]","[75,21]","[14.25,3.99]","[2016-12-03T11:41:17.000Z,2016-12-03T21:41:17.000Z]" +730372,2016-12-01T11:42:43.000Z,"[""Handbag - black/bronze"",""Jumpsuit - dark blue"",""2 PACK - Tights - black"",""High heeled sandals - avana""]","[""ZO0198001980"",""ZO0258702587"",""ZO0360703607"",""ZO0662306623""]","[25,65,11,85]","[4.75,12.35,2.09,16.15]","[2016-12-03T11:42:43.000Z,2016-12-03T21:42:43.000Z]" +551038,2016-12-01T11:47:02.000Z,"[""Bow tie - dark blue"",""Rucksack - rust cognac""]","[""ZO0409704097"",""ZO0604706047""]","[7,22]","[1.33,4.18]","[2016-12-03T11:47:02.000Z,2016-12-03T21:47:02.000Z]" +551042,2016-12-01T11:48:29.000Z,"[""Tights - grey multicolor/black"",""Ankle boots - dark brown""]","[""ZO0024100241"",""ZO0225802258""]","[25,33]","[4.75,6.27]","[2016-12-03T11:48:29.000Z,2016-12-03T21:48:29.000Z]" +725553,2016-12-01T11:58:34.000Z,"[""Tights - dark grey multicolor/coralle"",""Across body bag - Blue Violety"",""Sweatshirt - light grey multicolor"",""Long sleeved top - dark grey multicolor""]","[""ZO0096500965"",""ZO0180201802"",""ZO0222002220"",""ZO0640006400""]","[17,17,17,11]","[3.23,3.23,3.23,2.09]","[2016-12-03T11:58:34.000Z,2016-12-03T21:58:34.000Z]" +551064,2016-12-01T12:14:24.000Z,"[""Laptop bag - black"",""Belt - cognac ""]","[""ZO0315403154"",""ZO0598605986""]","[34,11]","[6.46,2.09]","[2016-12-03T12:14:24.000Z,2016-12-03T22:14:24.000Z]" +551068,2016-12-01T12:15:50.000Z,"[""Lace-ups - black"",""Lace-ups - marron""]","[""ZO0387003870"",""ZO0683906839""]","[60,60]","[11.4,11.4]","[2016-12-03T12:15:50.000Z,2016-12-03T22:15:50.000Z]" +551077,2016-12-01T12:24:29.000Z,"[""Tote bag - black"",""Lace-up boots - resin coffee""]","[""ZO0403504035"",""ZO0702207022""]","[42,50]","[7.98,9.5]","[2016-12-03T12:24:29.000Z,2016-12-03T22:24:29.000Z]" +551095,2016-12-01T12:38:53.000Z,"[""Trainers - black"",""Lace-up boots - cognac""]","[""ZO0404704047"",""ZO0509605096""]","[12,60]","[2.28,11.4]",[null] +551108,2016-12-01T12:56:10.000Z,"[""Vest - black"",""Boots - Light Grey""]","[""ZO0221202212"",""ZO0374503745""]","[12,100]","[2.28,19]","[2016-12-03T12:56:10.000Z,2016-12-03T22:56:10.000Z]" +551111,2016-12-01T12:59:02.000Z,"[""High heeled boots - Midnight Blue"",""Ankle boots - Olive Drab""]","[""ZO0361903619"",""ZO0374903749""]","[165,65]","[31.35,12.35]","[2016-12-03T12:59:02.000Z,2016-12-03T22:59:02.000Z]" +551114,2016-12-01T13:00:29.000Z,"[""Ballet pumps - rose-gold"",""Summer dress - black""]","[""ZO0337403374"",""ZO0365003650""]","[33,60]","[6.27,11.4]","[2016-12-03T13:00:29.000Z,2016-12-03T23:00:29.000Z]" +551115,2016-12-01T13:01:55.000Z,"[""Long sleeved top - navy/off-white/coral red"",""Sweatshirt - black""]","[""ZO0500905009"",""ZO0645706457""]","[12,21]","[2.28,3.99]","[2016-12-03T13:01:55.000Z,2016-12-03T23:01:55.000Z]" +551133,2016-12-01T13:26:24.000Z,"[""Vest - chilli pepper"",""Jersey dress - Blue Violety""]","[""ZO0042500425"",""ZO0062200622""]","[17,42]","[3.23,7.98]","[2016-12-03T13:26:24.000Z,2016-12-03T23:26:24.000Z]" +551137,2016-12-01T13:30:43.000Z,"[""3 PACK - Socks - black/grey"",""Cap - navy""]","[""ZO0460804608"",""ZO0664906649""]","[8,11]","[1.52,2.09]","[2016-12-03T13:30:43.000Z,2016-12-03T23:30:43.000Z]" +551141,2016-12-01T13:33:36.000Z,"[""Tracksuit top - dark grey multicolor"",""Swimming shorts - cactus""]","[""ZO0612406124"",""ZO0627906279""]","[33,21]","[6.27,3.99]","[2016-12-03T13:33:36.000Z,2016-12-03T23:33:36.000Z]" +551153,2016-12-01T13:42:14.000Z,"[""Shirt - red/black"",""Jumper - dark blue /offwhite""]","[""ZO0110201102"",""ZO0450304503""]","[34,33]","[6.46,6.27]",[2016-12-03T23:42:14.000Z] +551154,2016-12-01T13:42:14.000Z,"[""Briefcase - navy"",""Sports shirt - Seashell""]","[""ZO0466704667"",""ZO0617306173""]","[42,12]","[7.98,2.28]","[2016-12-03T13:42:14.000Z,2016-12-03T23:42:14.000Z]" +551183,2016-12-01T14:19:41.000Z,"[""Long sleeved top - mottled light grey/bordeaux"",""Print T-shirt - dark grey multicolor""]","[""ZO0439804398"",""ZO0546705467""]","[11,14]","[2.09,2.66]","[2016-12-03T14:19:41.000Z,2016-12-04T04:19:41.000Z]" +551185,2016-12-01T14:24:00.000Z,"[""Pyjamas - grey/blue"",""3 PACK - Shorts - grey""]","[""ZO0478404784"",""ZO0610306103""]","[29,13]","[5.51,2.47]",[2016-12-03T14:24:00.000Z] +551189,2016-12-01T14:28:19.000Z,"[""SLIM FIT - Formal shirt - lila"",""Snowboard jacket - pumpkin spice""]","[""ZO0422404224"",""ZO0624606246""]","[25,75]","[4.75,14.25]","[2016-12-03T14:28:19.000Z,2016-12-04T04:28:19.000Z]" +551194,2016-12-01T14:34:05.000Z,"[""Pencil skirt - white/red"",""Cardigan - black/offwhite""]","[""ZO0070500705"",""ZO0149101491""]","[17,33]","[3.23,6.27]","[2016-12-03T14:34:05.000Z,2016-12-04T04:34:05.000Z]" +551204,2016-12-01T14:44:10.000Z,"[""Bustier - white"",""Across body bag - cognac""]","[""ZO0200702007"",""ZO0212602126""]","[14,21]","[2.66,3.99]","[2016-12-03T14:44:10.000Z,2016-12-04T04:44:10.000Z]" +551220,2016-12-01T15:04:19.000Z,"[""Light jacket - dark blue"",""Tracksuit bottoms - dark blue""]","[""ZO0428304283"",""ZO0530105301""]","[50,25]","[9.5,4.75]","[2016-12-03T15:04:19.000Z,2016-12-04T05:04:19.000Z]" +551222,2016-12-01T15:05:46.000Z,"[""Chinos - Slim Fit - black"",""Basic T-shirt - rose""]","[""ZO0418904189"",""ZO0551805518""]","[23,8]","[4.37,1.52]","[2016-12-03T15:05:46.000Z,2016-12-04T05:05:46.000Z]" +729030,2016-12-01T15:10:05.000Z,"[""Ballet pumps - Dodger Blue"",""Across body bag - cognac"",""Platform heels - red"",""Wedges - beige""]","[""ZO0001800018"",""ZO0005200052"",""ZO0010100101"",""ZO0211402114""]","[21,31,32,15]","[3.99,5.89,6.08,2.85]","[2016-12-03T15:10:05.000Z,2016-12-04T05:10:05.000Z]" +726702,2016-12-01T15:21:36.000Z,"[""Classic heels - cognac"",""Jumper - dark grey"",""Summer dress - pomegranate"",""High heeled ankle boots - cognac""]","[""ZO0041700417"",""ZO0141101411"",""ZO0366503665"",""ZO0654206542""]","[55,19,33,42]","[10.45,3.61,6.27,7.98]","[2016-12-03T15:21:36.000Z,2016-12-04T05:21:36.000Z]" +551237,2016-12-01T15:24:29.000Z,"[""Scarf - black"",""Boots - dark brown""]","[""ZO0404504045"",""ZO0462704627""]","[13,60]","[2.47,11.4]","[2016-12-03T15:24:29.000Z,2016-12-04T05:24:29.000Z]" +551251,2016-12-01T15:36:00.000Z,"[""Print T-shirt - dark red"",""Print T-shirt - dark grey""]","[""ZO0494104941"",""ZO0494504945""]","[12,11]","[2.28,2.09]",[2016-12-03T15:36:00.000Z] +551271,2016-12-01T15:51:50.000Z,"[""Across body bag - brown"",""Jumper - dark blue""]","[""ZO0454004540"",""ZO0464104641""]","[15,33]","[2.85,6.27]",[2016-12-03T15:51:50.000Z] +551279,2016-12-01T16:03:22.000Z,"[""4 PACK - Shorts - black"",""Swimming shorts - dark blue""]","[""ZO0476304763"",""ZO0479304793""]","[17,21]","[3.23,3.99]","[2016-12-03T16:03:22.000Z,2016-12-04T06:03:22.000Z]" +551281,2016-12-01T16:04:48.000Z,"[""Handbag - cognac "",""Belt - brown""]","[""ZO0088900889"",""ZO0188001880""]","[25,11]","[4.75,2.09]",[2016-12-04T06:04:48.000Z] +551285,2016-12-01T16:07:41.000Z,"[""Sweatshirt - tan"",""T-bar sandals - black/green""]","[""ZO0522405224"",""ZO0588605886""]","[29,11]","[5.51,2.09]","[2016-12-03T16:07:41.000Z,2016-12-04T06:07:41.000Z]" diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/multivalue.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/multivalue.csv-spec new file mode 100644 index 0000000000000..e2721bead80e6 --- /dev/null +++ b/x-pack/plugin/sql/qa/server/src/main/resources/multivalue.csv-spec @@ -0,0 +1,102 @@ +// To mute tests follow example in file: example.csv-spec + +// +// H2 supports arrays, but without a keyword/function qualification +// + +simpleStar +SELECT * FROM ecommerce LIMIT 5; + + date | id | price | product | shipped | sku | tax +------------------------+---------------+---------------+---------------------------------+------------------------+---------------+------------------ +2016-12-01T00:04:19.000Z|550375 |11 |Basic T-shirt - Medium Slate Blue|2016-12-03T00:04:19.000Z|ZO0567505675 |2.0899999141693115 +2016-12-01T00:08:38.000Z|550385 |50 |Bomber Jacket - gold |2016-12-03T00:08:38.000Z|ZO0267102671 |9.5 +2016-12-01T00:23:02.000Z|550405 |60 |Trainers - rosa |2016-12-03T00:23:02.000Z|ZO0083400834 |11.399999618530273 +2016-12-01T00:33:07.000Z|550412 |21 |Trainers - white/black |2016-12-03T00:33:07.000Z|ZO0508905089 |3.990000009536743 +2016-12-01T00:47:31.000Z|550425 |65 |Boots - black |2016-12-03T00:47:31.000Z|ZO0256602566 |12.350000381469727 +; + +intColumnWithFiltering +SELECT id, ARRAY(sku) sku, ARRAY(price) price FROM ecommerce WHERE price < 10; + + id:i | sku:s_a | price:i_a +---------------+-------------------------------+--------------- +550522 |["ZO0258502585","ZO0481004810"]|[33,8] +550817 |["ZO0270302703","ZO0705107051"]|[8,65] +551038 |["ZO0409704097","ZO0604706047"]|[7,22] +551137 |["ZO0460804608","ZO0664906649"]|[8,11] +551222 |["ZO0418904189","ZO0551805518"]|[23,8] +; + +textColumnFiltering +SELECT id, ARRAY(product) product, ARRAY(price) price FROM ecommerce WHERE product LIKE 'Shirt%'; + + id | product | price +---------------+----------------------------------------------------+--------------- +550861 |["Parka - dark denim","Shirt - light blue denim"] |[65,33] +550864 |["Shorts - multicoloured","Shirt - grey/white"] |[21,25] +551153 |["Shirt - red/black","Jumper - dark blue /offwhite"]|[34,33] +; + +varyingMultiValueCount +SELECT id, date, ARRAY(price) price, ARRAY(sku) sku FROM ecommerce LIMIT 7; + + id | date | price | sku +---------------+------------------------+---------------+------------------------------------------------------------- +550375 |2016-12-01T00:04:19.000Z|[11,25] |["ZO0567505675","ZO0622006220"] +550385 |2016-12-01T00:08:38.000Z|[50,33] |["ZO0267102671","ZO0357703577"] +550405 |2016-12-01T00:23:02.000Z|[60,12] |["ZO0083400834","ZO0240802408"] +550412 |2016-12-01T00:33:07.000Z|[21,60] |["ZO0508905089","ZO0681206812"] +550425 |2016-12-01T00:47:31.000Z|[65,21] |["ZO0256602566","ZO0516305163"] +728387 |2016-12-01T00:50:24.000Z|[21,50,75,25] |["ZO0089900899","ZO0205902059","ZO0242802428","ZO0338503385"] +550428 |2016-12-01T00:54:43.000Z|[22,42] |["ZO0195801958","ZO0303703037"] +; + +dateColumnFiltering +SELECT id, ARRAY(sku), ARRAY(shipped) FROM ecommerce WHERE shipped > '2016-12-04T00:00:00Z'::DATETIME; + + id | ARRAY(sku) | ARRAY(shipped) +---------------+-------------------------------------------------------------+--------------------------------------------------- +551183 |["ZO0439804398","ZO0546705467"] |[2016-12-03T14:19:41.000Z,2016-12-04T04:19:41.000Z] +551189 |["ZO0422404224","ZO0624606246"] |[2016-12-03T14:28:19.000Z,2016-12-04T04:28:19.000Z] +551194 |["ZO0070500705","ZO0149101491"] |[2016-12-03T14:34:05.000Z,2016-12-04T04:34:05.000Z] +551204 |["ZO0200702007","ZO0212602126"] |[2016-12-03T14:44:10.000Z,2016-12-04T04:44:10.000Z] +551220 |["ZO0428304283","ZO0530105301"] |[2016-12-03T15:04:19.000Z,2016-12-04T05:04:19.000Z] +551222 |["ZO0418904189","ZO0551805518"] |[2016-12-03T15:05:46.000Z,2016-12-04T05:05:46.000Z] +729030 |["ZO0001800018","ZO0005200052","ZO0010100101","ZO0211402114"]|[2016-12-03T15:10:05.000Z,2016-12-04T05:10:05.000Z] +726702 |["ZO0041700417","ZO0141101411","ZO0366503665","ZO0654206542"]|[2016-12-03T15:21:36.000Z,2016-12-04T05:21:36.000Z] +551237 |["ZO0404504045","ZO0462704627"] |[2016-12-03T15:24:29.000Z,2016-12-04T05:24:29.000Z] +551279 |["ZO0476304763","ZO0479304793"] |[2016-12-03T16:03:22.000Z,2016-12-04T06:03:22.000Z] +551281 |["ZO0088900889","ZO0188001880"] |[2016-12-04T06:04:48.000Z] +551285 |["ZO0522405224","ZO0588605886"] |[2016-12-03T16:07:41.000Z,2016-12-04T06:07:41.000Z] +; + +nullsListing +SELECT id, shipped, ARRAY(shipped) FROM ecommerce WHERE shipped IS NULL OR id IN (550578, 550694, 550710, 550736) ORDER BY shipped NULLS FIRST; + + id:i | shipped:ts | ARRAY(shipped):ts_a +---------------+------------------------+-------------------------- +550503 |null |[] +550659 |null |[] +550784 |null |[] +550810 |null |[] +550833 |null |[] +551095 |null |[] +550694 |2016-12-03T05:42:43.000Z|[2016-12-03T05:42:43.000Z] +550710 |2016-12-03T06:00:00.000Z|[2016-12-03T06:00:00.000Z] +550578 |2016-12-03T13:38:53.000Z|[2016-12-03T13:38:53.000Z] +550736 |2016-12-03T16:27:22.000Z|[2016-12-03T16:27:22.000Z] + +; + +scalarPlusMultiValue +SELECT id, price, ARRAY(price), sku, ARRAY(sku) FROM ecommerce LIMIT 5; + + id | price | ARRAY(price) | sku | ARRAY(sku) +---------------+---------------+--------------+---------------+------------------------------- +550375 |11 |[11,25] |ZO0567505675 |["ZO0567505675","ZO0622006220"] +550385 |50 |[50,33] |ZO0267102671 |["ZO0267102671","ZO0357703577"] +550405 |60 |[60,12] |ZO0083400834 |["ZO0083400834","ZO0240802408"] +550412 |21 |[21,60] |ZO0508905089 |["ZO0508905089","ZO0681206812"] +550425 |65 |[65,21] |ZO0256602566 |["ZO0256602566","ZO0516305163"] +; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec index 95d3d46c45640..b0ad393d83ba8 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/single-node-only/command-sys.csv-spec @@ -95,6 +95,15 @@ SYS COLUMNS TABLE LIKE '%'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s -----------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |ecommerce |date |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |price |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |product |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |product.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |shipped |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |sku |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |sku.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |ecommerce |tax |8 |DOUBLE |25 |8 |null |2 |1 |null |null |8 |0 |null |9 |YES |null |null |null |null |NO |NO integTest |null |logs |@timestamp |93 |DATETIME |34 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec index 71ec21a82687e..b534a56951951 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/slow/frozen.csv-spec @@ -7,7 +7,8 @@ showTables SHOW TABLES INCLUDE FROZEN; - name | type | kind + name | type | kind +ecommerce |TABLE |INDEX frozen_emp |TABLE |FROZEN INDEX logs |TABLE |INDEX logs_nanos |TABLE |INDEX diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java index d3e14f5a00a52..acf09a10de201 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java @@ -72,14 +72,14 @@ public static SqlQueryResponse createRandomInstance(String cursor, Mode mode, bo rows = new ArrayList<>(rowCount); for (int r = 0; r < rowCount; r++) { - List row = new ArrayList<>(rowCount); + List row = new ArrayList<>(columnCount); for (int c = 0; c < columnCount; c++) { Supplier value = randomFrom(Arrays.asList( () -> randomAlphaOfLength(10), ESTestCase::randomLong, ESTestCase::randomDouble, () -> null)); - row.add(value.get()); + row.add(randomBoolean() ? value.get() : randomList(randomIntBetween(1, 10), value)); } rows.add(row); } diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryResponse.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryResponse.java index 167d8542cccb0..7414feaa30ccd 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryResponse.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/SqlQueryResponse.java @@ -95,13 +95,35 @@ public static List> parseRows(XContentParser parser) throws IOExcep public static List parseRow(XContentParser parser) throws IOException { List list = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - if (parser.currentToken().isValue()) { - list.add(ProtoUtils.parseFieldsValue(parser)); - } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { - list.add(null); - } else { - throw new IllegalStateException("expected value but got [" + parser.currentToken() + "]"); + List savedList = null; + for (boolean parsing = true; parsing; ) { + switch (parser.nextToken()) { + case END_ARRAY: + if (savedList != null) { + savedList.add(list); + list = savedList; + savedList = null; + } else { + parsing = false; + } + break; + case START_ARRAY: + if (savedList == null) { + savedList = list; + list = new ArrayList<>(); + } else { + throw new IllegalStateException("multidimensional multivalue not supported"); + } + break; + case VALUE_NULL: + list.add(null); + break; + default: + if (parser.currentToken().isValue()) { + list.add(ProtoUtils.parseFieldsValue(parser)); + } else { + throw new IllegalStateException("expected value but got [" + parser.currentToken() + "]"); + } } } return list; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 41a2be89b034f..955c5fce79b1b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -56,6 +56,7 @@ import org.elasticsearch.xpack.sql.plan.logical.LocalRelation; import org.elasticsearch.xpack.sql.plan.logical.Pivot; import org.elasticsearch.xpack.sql.plan.logical.command.Command; +import org.elasticsearch.xpack.sql.proto.SqlVersion; import org.elasticsearch.xpack.sql.stats.FeatureMetric; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.type.SqlDataTypes; @@ -76,6 +77,8 @@ import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.versionIntroducingType; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -92,9 +95,11 @@ */ public final class Verifier { private final Metrics metrics; + private final SqlVersion version; - public Verifier(Metrics metrics) { + public Verifier(Metrics metrics, SqlVersion version) { this.metrics = metrics; + this.version = version; } public Map, String> verifyFailures(LogicalPlan plan) { @@ -236,6 +241,8 @@ Collection verify(LogicalPlan plan) { failures.addAll(localFailures); }); + + checkClientSupportsDataTypes(plan, failures, version); } // gather metrics @@ -915,4 +922,14 @@ private static void checkArrayFunctionArguments(LogicalPlan plan, Set l } })); } + + private static void checkClientSupportsDataTypes(LogicalPlan p, Set localFailures, SqlVersion version) { + p.output().forEach(e -> { + if (e.resolved() && isTypeSupportedInVersion(e.dataType(), version) == false) { + localFailures.add(fail(e, "Cannot use [" + e.name() + "] with type [" + e.dataType() + "] unsupported " + + "in version [" + version + "], upgrade required (to version [" + versionIntroducingType(e.dataType()) + + "] or higher)")); + } + }); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java index b5a7ccb1194f4..b5e2d1e5f519d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java @@ -42,7 +42,6 @@ public class PlanExecutor { private final IndexResolver indexResolver; private final PreAnalyzer preAnalyzer; - private final Verifier verifier; private final Optimizer optimizer; private final Planner planner; @@ -58,13 +57,13 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe this.metrics = new Metrics(); this.preAnalyzer = new PreAnalyzer(); - this.verifier = new Verifier(metrics); this.optimizer = new Optimizer(); this.planner = new Planner(); } private SqlSession newSession(SqlConfiguration cfg) { - return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, verifier, optimizer, planner, this); + return new SqlSession(cfg, client, functionRegistry, indexResolver, preAnalyzer, new Verifier(metrics, cfg.version()), optimizer, + planner, this); } public void searchSource(SqlConfiguration cfg, String sql, List params, diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java index e99bff4f9e73f..0901299cdcd34 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypes.java @@ -28,6 +28,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.SHORT; import static org.elasticsearch.xpack.ql.type.DataTypes.isSigned; import static org.elasticsearch.xpack.ql.type.DataTypes.isString; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDataType; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlDateTimeSub; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.metaSqlMaximumScale; @@ -82,7 +83,8 @@ public List output() { @Override public final void execute(SqlSession session, ActionListener listener) { - Stream values = SqlDataTypes.types().stream(); + Stream values = SqlDataTypes.types().stream() + .filter(t -> isTypeSupportedInVersion(t, session.configuration().version())); if (type.intValue() != 0) { values = values.filter(t -> type.equals(sqlType(t).getVendorTypeNumber())); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index 442390e9981fc..ec84c2beec0f8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -78,7 +78,7 @@ public RestResponse buildResponse(SqlQueryResponse response) throws Exception { response.toXContent(builder, request); restResponse = new BytesRestResponse(RestStatus.OK, builder); } else { // TextFormat - TextFormat type = (TextFormat)responseMediaType; + TextFormat type = (TextFormat) responseMediaType; final String data = type.format(request, response); restResponse = new BytesRestResponse(RestStatus.OK, type.contentType(request), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java new file mode 100644 index 0000000000000..4e95761cee53a --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.session; + +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.sql.proto.SqlVersion; + +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.Version.V_7_13_0; +import static org.elasticsearch.xpack.ql.type.DataTypes.isArray; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.types; + +public final class VersionCompatibilityChecks { + + public static final SqlVersion INTRODUCING_ARRAY_TYPES = SqlVersion.fromId(V_7_13_0.id); + + private static final Map TYPE_TO_VERSION_MAP = new HashMap<>(); + + static { + types().stream().filter(DataTypes::isArray).forEach(x -> TYPE_TO_VERSION_MAP.put(x, INTRODUCING_ARRAY_TYPES)); + // Note: future new array types (introduced with new ES types) will require a mapping update here below. + } + + private VersionCompatibilityChecks() {} + + /** + * Is the provided {@code dataType} being supported in the provided {@code version}? + */ + public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion version) { + if (isArray(dataType)) { + return supportsArrayTypes(version); + } + return true; + } + /** + * Does the provided {@code version} support the array types? + */ + public static boolean supportsArrayTypes(SqlVersion version) { + // TODO: add equality only once actually ported to target branch + return version.compareTo(INTRODUCING_ARRAY_TYPES) > 0; + } + + public static SqlVersion versionIntroducingType(DataType type) { + return TYPE_TO_VERSION_MAP.get(type); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java index 9fa54bc107e7b..74e5ae60d3b9b 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/FieldAttributeTests.java @@ -25,6 +25,8 @@ import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; import org.elasticsearch.xpack.sql.parser.SqlParser; +import org.elasticsearch.xpack.sql.proto.SqlVersion; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.stats.Metrics; import java.util.List; @@ -34,6 +36,8 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_ARRAY_TYPES; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.LONG_ARRAY; import static org.elasticsearch.xpack.sql.types.SqlTypesTests.loadMapping; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.contains; @@ -54,7 +58,7 @@ public class FieldAttributeTests extends ESTestCase { public FieldAttributeTests() { parser = new SqlParser(); functionRegistry = new SqlFunctionRegistry(); - verifier = new Verifier(new Metrics()); + verifier = new Verifier(new Metrics(), SqlTestUtils.TEST_CFG.version()); Map mapping = loadMapping("mapping-multi-field-variation.json"); @@ -279,6 +283,56 @@ public void testGroupByAmbiguity() { ex.getMessage()); } + public void testArrayTypeVersionCompatibility() { + Map mapping = TypesTests.loadMapping("mapping-numeric.json"); + EsIndex index = new EsIndex("test", mapping); + getIndexResult = IndexResolution.valid(index); + + + SqlVersion preArrayTypes = SqlVersion.fromId(INTRODUCING_ARRAY_TYPES.id - SqlVersion.MINOR_MULTIPLIER); + SqlVersion postArrayTypes = SqlVersion.fromId(INTRODUCING_ARRAY_TYPES.id + SqlVersion.MINOR_MULTIPLIER); + + String query = "SELECT ARRAY(long) FROM test"; + + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preArrayTypes); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); + VerificationException ex = expectThrows(VerificationException.class, () -> plan(query)); + assertEquals( + "Found 1 problem\nline 1:8: Cannot use [ARRAY(long)] with type [LONG_ARRAY] unsupported in version [" + + preArrayTypes + "], upgrade required (to version [" + INTRODUCING_ARRAY_TYPES + "] or higher)", + ex.getMessage()); + + for (SqlVersion v : List.of(INTRODUCING_ARRAY_TYPES, postArrayTypes)) { + analyzer = new Analyzer(SqlTestUtils.randomConfiguration(v), functionRegistry, getIndexResult, verifier); + LogicalPlan plan = plan(query); + assertThat(plan, instanceOf(Project.class)); + Project p = (Project) plan; + List projections = p.projections(); + assertThat(projections, hasSize(1)); + Attribute attribute = projections.get(0).toAttribute(); + assertThat(attribute.dataType(), is(LONG_ARRAY)); + assertThat(attribute.name(), is("ARRAY(long)")); + } + } + + public void testUnprojectedArrayTypeVersionCompatibility() { + Map mapping = TypesTests.loadMapping("mapping-numeric.json"); + EsIndex index = new EsIndex("test", mapping); + getIndexResult = IndexResolution.valid(index); + SqlVersion preArrayTypes = SqlVersion.fromId(INTRODUCING_ARRAY_TYPES.id - SqlVersion.MINOR_MULTIPLIER); + SqlConfiguration sqlConfig = SqlTestUtils.randomConfiguration(preArrayTypes); + analyzer = new Analyzer(sqlConfig, functionRegistry, getIndexResult, new Verifier(new Metrics(), sqlConfig.version())); + + String query = "SELECT l = 1 FROM (SELECT ARRAY(long), long AS l FROM test WHERE l > 10)"; + + LogicalPlan plan = plan(query); + assertThat(plan, instanceOf(Project.class)); + Project p = (Project) plan; + List projections = p.projections(); + assertThat(projections, hasSize(1)); + assertEquals(projections.get(0).dataType(), BOOLEAN); + } + public void testFunctionOverNonExistingFieldAsArgumentAndSameAlias() throws Exception { Map mapping = TypesTests.loadMapping("mapping-basic.json"); EsIndex index = new EsIndex("test", mapping); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index ce380848aebff..afb37d570fb58 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -51,7 +51,8 @@ private String error(String sql) { } private String error(IndexResolution getIndexResult, String sql) { - Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics(), + TEST_CFG.version())); VerificationException e = expectThrows(VerificationException.class, () -> analyzer.analyze(parser.createStatement(sql), true)); String message = e.getMessage(); assertTrue(message.startsWith("Found ")); @@ -71,7 +72,8 @@ private EsIndex getTestEsIndex() { } private LogicalPlan accept(IndexResolution resolution, String sql) { - Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), resolution, new Verifier(new Metrics(), + TEST_CFG.version())); return analyzer.analyze(parser.createStatement(sql), true); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index 1daa455b5e5e5..ae0ec79cbbaec 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -144,7 +144,8 @@ public void testMultiValuedDocValue() { DocumentField field = new DocumentField(fieldName, asList("a", "b")); SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); + assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + + "use ARRAY(" + fieldName + ") instead")); } public void testExtractSourcePath() { @@ -154,16 +155,16 @@ public void testExtractSourcePath() { SearchHit hit = new SearchHit(1, null, null, singletonMap("a.b.c", field), null); assertThat(fe.extract(hit), is(value)); } - + public void testMultiValuedSource() { FieldHitExtractor fe = getFieldHitExtractor("a"); Object value = randomValue(); DocumentField field = new DocumentField("a", asList(value, value)); SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Arrays (returned by [a]) are not supported")); + assertThat(ex.getMessage(), is("Cannot return multiple values for field [a]; use ARRAY(a) instead")); } - + public void testMultiValuedSourceAllowed() { FieldHitExtractor fe = new FieldHitExtractor("a", null, UTC, EXTRACT_ONE); Object valueA = randomValue(); @@ -173,8 +174,8 @@ public void testMultiValuedSourceAllowed() { assertEquals(valueA, fe.extract(hit)); } - // "a": null - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") + // "a": null -> [null] + // note though that through fields API an empty list will be returned for missing field values, not null public void testMultiValueNull() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fea = getArrayFieldHitExtractor(fieldName, INTEGER); @@ -182,17 +183,15 @@ public void testMultiValueNull() { new DocumentField(fieldName, singletonList(null))), null))); } - // "a": [] / missing - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") + // "a": [] / missing -> [] public void testMultiValueEmpty() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fea = getArrayFieldHitExtractor(fieldName, INTEGER); - assertEquals(singletonList(null), fea.extract(new SearchHit(1, null, null, singletonMap(fieldName, + assertEquals(emptyList(), fea.extract(new SearchHit(1, null, null, singletonMap(fieldName, new DocumentField(fieldName, emptyList())), null))); } // "a": [int1, int2, ..] - @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testMultiValueImmediateFromMap() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fea = getArrayFieldHitExtractor(fieldName, INTEGER); @@ -213,11 +212,11 @@ public void testGeoShapeExtraction() { assertEquals(new GeoShape(1, 2), fe.extract(hit)); } - + public void testMultipleGeoShapeExtraction() { String fieldName = randomAlphaOfLength(5); FieldHitExtractor fe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, FAIL_IF_MULTIVALUE); - + Map map1 = new HashMap<>(2); map1.put("coordinates", asList(1d, 2d)); map1.put("type", "Point"); @@ -228,11 +227,17 @@ public void testMultipleGeoShapeExtraction() { SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); - + assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + + "use ARRAY(" + fieldName + ") instead")); + FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, EXTRACT_ONE); assertEquals(new GeoShape(3, 4), lenientFe.extract(new SearchHit(1, null, null, singletonMap(fieldName, new DocumentField(fieldName, singletonList(map2))), null))); + + FieldHitExtractor arrayFe = getArrayFieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE); + assertEquals(asList(new GeoShape(1, 2), new GeoShape(3, 4)), arrayFe.extract(new SearchHit(1, null, null, singletonMap(fieldName, + new DocumentField(fieldName, asList(map1, map2))), null))); + } private FieldHitExtractor getFieldHitExtractor(String fieldName) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java index a517ea98d7e51..cf08ce1342409 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/DatabaseFunctionTests.java @@ -38,7 +38,7 @@ public void testDatabaseFunctionOutput() { sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics()) + new Verifier(new Metrics(), sqlConfig.version()) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT DATABASE()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java index a4ff40f177a11..7873cca35e893 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/UserFunctionTests.java @@ -38,7 +38,7 @@ null, null, randomAlphaOfLengthBetween(1, 15), sqlConfig, new SqlFunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics()) + new Verifier(new Metrics(), sqlConfig.version()) ); Project result = (Project) analyzer.analyze(parser.createStatement("SELECT USER()"), true); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java index 996f8c9ba6faa..625ce63e85244 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java @@ -28,6 +28,7 @@ import java.util.Objects; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.sql.SqlTestUtils.literal; public class CurrentDateTimeTests extends AbstractNodeTestCase { @@ -92,7 +93,8 @@ public void testInvalidPrecision() { IndexResolution indexResolution = IndexResolution.valid(new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), indexResolution, + new Verifier(new Metrics(), TEST_CFG.version())); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIMESTAMP(100000000000000)"), true)); assertEquals("line 1:27: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java index 24dbe8db22f70..2ee6a64d52efc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentTimeTests.java @@ -29,6 +29,7 @@ import java.util.Objects; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.sql.SqlTestUtils.literal; public class CurrentTimeTests extends AbstractNodeTestCase { @@ -93,7 +94,8 @@ public void testInvalidPrecision() { IndexResolution indexResolution = IndexResolution.valid(new EsIndex("test", SqlTypesTests.loadMapping("mapping-multi-field-with-nested.json"))); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), indexResolution, new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), indexResolution, + new Verifier(new Metrics(), TEST_CFG.version())); ParsingException e = expectThrows(ParsingException.class, () -> analyzer.analyze(parser.createStatement("SELECT CURRENT_TIME(100000000000000)"), true)); assertEquals("line 1:22: invalid precision; [100000000000000] out of [integer] range", e.getMessage()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java index 43b8bd12999dd..76e0adfb7fc69 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/optimizer/OptimizerRunTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.type.EsField; -import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; import org.elasticsearch.xpack.sql.parser.SqlParser; @@ -48,6 +47,7 @@ import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.LTE; import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.NEQ; import static org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation.NULLEQ; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; public class OptimizerRunTests extends ESTestCase { @@ -77,7 +77,8 @@ public OptimizerRunTests() { EsIndex test = new EsIndex("test", mapping); getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, functionRegistry, getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(TEST_CFG, functionRegistry, getIndexResult, + new Verifier(new Metrics(), TEST_CFG.version())); optimizer = new Optimizer(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index 5a0b354c603a9..a4e9b34e3a268 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -271,7 +271,8 @@ private void executeCommand(String sql, List params, private Tuple sql(String sql, List params, SqlConfiguration config, Map mapping) { EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(config, new FunctionRegistry(), IndexResolution.valid(test), + new Verifier(new Metrics(), config.version())); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, UTC), true); IndexResolver resolver = mock(IndexResolver.class); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 0cb38a2ae5319..4776bd558b0de 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.ql.index.IndexResolver.IndexType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.EsField; -import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; import org.elasticsearch.xpack.sql.parser.SqlParser; @@ -25,8 +24,8 @@ import org.elasticsearch.xpack.sql.proto.Mode; import org.elasticsearch.xpack.sql.proto.Protocol; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; -import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.SchemaRowSet; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.types.SqlTypesTests; @@ -44,6 +43,7 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.ql.index.IndexResolver.SQL_TABLE; import static org.elasticsearch.xpack.ql.index.IndexResolver.SQL_VIEW; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -334,8 +334,8 @@ private SqlTypedParamValue param(Object value) { private Tuple sql(String sql, List params, SqlConfiguration cfg) { EsIndex test = new EsIndex("test", mapping); - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), - new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), + new Verifier(new Metrics(), TEST_CFG.version())); Command cmd = (Command) analyzer.analyze(parser.createStatement(sql, params, cfg.zoneId()), true); IndexResolver resolver = mock(IndexResolver.class); @@ -355,7 +355,7 @@ private void executeCommand(String sql, Consumer consumer, SqlConf private void executeCommand(String sql, List params, Consumer consumer, IndexInfo... infos) throws Exception { - executeCommand(sql, params, consumer, SqlTestUtils.TEST_CFG, infos); + executeCommand(sql, params, consumer, TEST_CFG, infos); } @SuppressWarnings({ "unchecked", "rawtypes" }) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 4336d16c17bf4..20b522e30de49 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.index.IndexResolver; +import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.parser.SqlParser; @@ -23,18 +24,22 @@ import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.SqlSession; -import org.elasticsearch.xpack.sql.type.SqlDataTypes; import org.elasticsearch.xpack.sql.types.SqlTypesTests; import org.elasticsearch.xpack.sql.util.DateUtils; import java.sql.JDBCType; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Arrays.asList; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.INTRODUCING_ARRAY_TYPES; +import static org.elasticsearch.xpack.sql.session.VersionCompatibilityChecks.isTypeSupportedInVersion; +import static org.elasticsearch.xpack.sql.type.SqlDataTypes.types; import static org.mockito.Mockito.mock; public class SysTypesTests extends ESTestCase { @@ -82,7 +87,7 @@ public void testSysTypes() { cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); assertEquals(19, r.columnCount()); - assertEquals(SqlDataTypes.types().size(), r.size()); + assertEquals(types().size(), r.size()); assertFalse(r.schema().types().contains(DataTypes.NULL)); // test first numeric (i.e. BYTE) as signed assertFalse(r.column(9, Boolean.class)); @@ -104,7 +109,7 @@ public void testSysTypesDefaultFiltering() { cmd.v1().execute(cmd.v2(), wrap(p -> { SchemaRowSet r = (SchemaRowSet) p.rowSet(); - assertEquals(SqlDataTypes.types().size(), r.size()); + assertEquals(types().size(), r.size()); }, ex -> fail(ex.getMessage()))); } @@ -142,4 +147,33 @@ public void testSysTypesMultipleMatches() { assertEquals("TEXT", r.column(0)); }, ex -> fail(ex.getMessage()))); } + + public void testArrayTypesFiltering() { + Set versions = new HashSet<>(List.of( + SqlVersion.fromId(INTRODUCING_ARRAY_TYPES.id - SqlVersion.MINOR_MULTIPLIER), + INTRODUCING_ARRAY_TYPES, + SqlVersion.fromId(INTRODUCING_ARRAY_TYPES.id + SqlVersion.MINOR_MULTIPLIER), + SqlVersion.fromId(Version.CURRENT.id) + )); + versions.add(null); + + for (SqlVersion version : versions) { + for (Mode mode : Mode.values()) { + Tuple cmd = sql("SYS TYPES", mode, version); + SqlSession session = cmd.v2(); + + cmd.v1().execute(session, wrap(p -> { + List types = new ArrayList<>(); + + SchemaRowSet r = (SchemaRowSet) p.rowSet(); + r.forEachRow(rv -> types.add((String) rv.column(0))); + + for (DataType arrayType : types().stream().filter(DataTypes::isArray).collect(Collectors.toList())) { + assertEquals(isTypeSupportedInVersion(arrayType, session.configuration().version()), + types.contains(arrayType.toString())); + } + }, ex -> fail(ex.getMessage()))); + } + } + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java index 88434f3da1075..2a0dc3374a776 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryFolderTests.java @@ -35,6 +35,7 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.startsWith; @@ -53,7 +54,8 @@ public static void init() { Map mapping = SqlTypesTests.loadMapping("mapping-multi-field-variation.json"); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), getIndexResult, + new Verifier(new Metrics(), TEST_CFG.version())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index fd0d46cf67d24..b076d04a407c2 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -136,7 +136,7 @@ private static class TestContext { Map mapping = SqlTypesTests.loadMapping(mappingFile); EsIndex test = new EsIndex("test", mapping); IndexResolution getIndexResult = IndexResolution.valid(test); - analyzer = new Analyzer(TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics())); + analyzer = new Analyzer(TEST_CFG, sqlFunctionRegistry, getIndexResult, new Verifier(new Metrics(), TEST_CFG.version())); optimizer = new Optimizer(); planner = new Planner(); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java index 263fe373e2e38..890fa2a72ba93 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/stats/VerifierMetricsTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.type.EsField; -import org.elasticsearch.xpack.sql.SqlTestUtils; import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer; import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier; import org.elasticsearch.xpack.sql.expression.function.SqlFunctionRegistry; @@ -21,6 +20,7 @@ import java.util.Map; +import static org.elasticsearch.xpack.sql.SqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.COMMAND; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.GROUPBY; import static org.elasticsearch.xpack.sql.stats.FeatureMetric.HAVING; @@ -161,7 +161,7 @@ public void testWhereLimitGroupByHavingOrderByQuery() { public void testTwoQueriesExecuted() { Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics); + Verifier verifier = new Verifier(metrics, TEST_CFG.version()); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages LIMIT 5", verifier); sqlWithVerifier("SELECT languages FROM test WHERE languages > 2 GROUP BY languages HAVING MAX(languages) > 3 " + "ORDER BY languages LIMIT 5", verifier); @@ -179,7 +179,7 @@ public void testTwoQueriesExecuted() { public void testTwoCommandsExecuted() { String command1 = randomFrom(commands); Metrics metrics = new Metrics(); - Verifier verifier = new Verifier(metrics); + Verifier verifier = new Verifier(metrics, TEST_CFG.version()); sqlWithVerifier(command1, verifier); sqlWithVerifier(randomValueOtherThan(command1, () -> randomFrom(commands)), verifier); Counters c = metrics.stats(); @@ -237,10 +237,10 @@ private Counters sql(String sql, Verifier v) { Metrics metrics = null; if (v == null) { metrics = new Metrics(); - verifier = new Verifier(metrics); + verifier = new Verifier(metrics, TEST_CFG.version()); } - Analyzer analyzer = new Analyzer(SqlTestUtils.TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier); + Analyzer analyzer = new Analyzer(TEST_CFG, new SqlFunctionRegistry(), IndexResolution.valid(test), verifier); analyzer.analyze(parser.createStatement(sql), true); return metrics == null ? null : metrics.stats();