Skip to content

Commit 0d40d65

Browse files
bpinteabpintea
authored
SQL: Add text formatting support for multivalue (#68606)
* Add text formatting support for multivalue This adds the text formatting for multivalue doc fields, that is: - CSV and TSV exports - TXT formatting
1 parent 057118b commit 0d40d65

File tree

4 files changed

+211
-7
lines changed

4 files changed

+211
-7
lines changed

‎x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,28 @@ private void executeQueryWithNextPage(String format, String expectedHeader, Stri
10991099
assertEquals(0, getNumberOfSearchContexts(client(), "test"));
11001100
}
11011101

1102+
public void testMultiValueQueryText() throws IOException {
1103+
index(
1104+
"{"
1105+
+ toJson("text")
1106+
+ ":["
1107+
+ toJson("one")
1108+
+ ","
1109+
+ toJson("two, three")
1110+
+ ","
1111+
+ toJson("\"four\"")
1112+
+ "], "
1113+
+ toJson("number")
1114+
+ " : [1, [2, 3], 4] }"
1115+
);
1116+
1117+
String expected = " t | n \n"
1118+
+ "-------------------------------+---------------\n"
1119+
+ "[\"one\",\"two, three\",\"\\\"four\\\"\"]|[1,2,3,4] \n";
1120+
Tuple<String, String> response = runSqlAsText("SELECT ARRAY(text) t, ARRAY(number) n FROM test", "text/plain");
1121+
assertEquals(expected, response.v1());
1122+
}
1123+
11021124
private Tuple<String, String> runSqlAsText(String sql, String accept) throws IOException {
11031125
return runSqlAsText(StringUtils.EMPTY, new StringEntity(query(sql).toString(), ContentType.APPLICATION_JSON), accept);
11041126
}

‎x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/StringUtils.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@
1414
import java.time.ZonedDateTime;
1515
import java.time.format.DateTimeFormatter;
1616
import java.time.format.DateTimeFormatterBuilder;
17+
import java.util.Collection;
1718
import java.util.Locale;
1819
import java.util.Objects;
20+
import java.util.StringJoiner;
1921
import java.util.concurrent.TimeUnit;
2022

2123
import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
@@ -166,6 +168,23 @@ public static String toString(Object value, SqlVersion sqlVersion) {
166168
return sb.toString();
167169
}
168170

171+
// multivalue
172+
if (value instanceof Collection) {
173+
final StringJoiner sj = new StringJoiner(",", "[", "]");
174+
Collection<?> values = (Collection<?>) value;
175+
values.forEach(x -> {
176+
// quote strings and `\`-escape the `"` character inside them
177+
if (x instanceof String) { // TODO: IPs should ideally not be quoted.
178+
sj.add('"' + ((String) x).replace("\"", "\\\"") + '"');
179+
} else if (x == null) {
180+
sj.add("NULL");
181+
} else {
182+
sj.add(toString(x, sqlVersion));
183+
}
184+
});
185+
return sj.toString();
186+
}
187+
169188
return Objects.toString(value);
170189
}
171190

‎x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,20 @@
1010
import org.elasticsearch.common.collect.Tuple;
1111
import org.elasticsearch.common.xcontent.MediaType;
1212
import org.elasticsearch.rest.RestRequest;
13-
import org.elasticsearch.xpack.ql.util.StringUtils;
1413
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
1514
import org.elasticsearch.xpack.sql.action.BasicFormatter;
1615
import org.elasticsearch.xpack.sql.action.SqlQueryResponse;
1716
import org.elasticsearch.xpack.sql.proto.ColumnInfo;
17+
import org.elasticsearch.xpack.sql.proto.StringUtils;
1818
import org.elasticsearch.xpack.sql.session.Cursor;
1919
import org.elasticsearch.xpack.sql.session.Cursors;
20-
import org.elasticsearch.xpack.sql.util.DateUtils;
2120

2221
import java.net.URLDecoder;
2322
import java.nio.charset.StandardCharsets;
2423
import java.time.ZoneId;
25-
import java.time.ZonedDateTime;
2624
import java.util.List;
2725
import java.util.Locale;
2826
import java.util.Map;
29-
import java.util.Objects;
3027
import java.util.Set;
3128
import java.util.function.Function;
3229

@@ -195,7 +192,7 @@ String maybeEscape(String value, Character delimiter) {
195192
sb.append('"');
196193
for (int i = 0; i < value.length(); i++) {
197194
char c = value.charAt(i);
198-
if (value.charAt(i) == '"') {
195+
if (c == '"') {
199196
sb.append('"');
200197
}
201198
sb.append(c);
@@ -319,8 +316,7 @@ String format(RestRequest request, SqlQueryResponse response) {
319316
}
320317

321318
for (List<Object> row : response.rows()) {
322-
row(sb, row, f -> f instanceof ZonedDateTime ? DateUtils.toString((ZonedDateTime) f) : Objects.toString(f, StringUtils.EMPTY),
323-
delimiter(request));
319+
row(sb, row, f -> f == null ? StringUtils.EMPTY : StringUtils.toString(f), delimiter(request));
324320
}
325321

326322
return sb.toString();

‎x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plugin/TextFormatTests.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
*/
77
package org.elasticsearch.xpack.sql.plugin;
88

9+
import java.time.ZonedDateTime;
910
import java.util.ArrayList;
1011
import java.util.Arrays;
12+
import java.util.Collection;
1113
import java.util.List;
1214
import java.util.Set;
1315
import java.util.stream.Collectors;
@@ -17,6 +19,7 @@
1719
import org.elasticsearch.test.ESTestCase;
1820
import org.elasticsearch.test.rest.FakeRestRequest;
1921
import org.elasticsearch.xpack.sql.action.SqlQueryResponse;
22+
import org.elasticsearch.xpack.sql.expression.literal.geo.GeoShape;
2023
import org.elasticsearch.xpack.sql.proto.ColumnInfo;
2124
import org.elasticsearch.xpack.sql.proto.Mode;
2225

@@ -27,6 +30,7 @@
2730
import static org.elasticsearch.xpack.sql.plugin.TextFormat.CSV;
2831
import static org.elasticsearch.xpack.sql.plugin.TextFormat.TSV;
2932
import static org.elasticsearch.xpack.sql.proto.SqlVersion.DATE_NANOS_SUPPORT_VERSION;
33+
import static org.elasticsearch.xpack.sql.type.SqlDataTypes.fromJava;
3034

3135
public class TextFormatTests extends ESTestCase {
3236

@@ -152,6 +156,151 @@ public void testInvalidCsvDelims() {
152156
}
153157
}
154158

159+
public void testCsvMultiValueEmpty() {
160+
String text = CSV.format(req(), withData(singletonList(singletonList(emptyList()))));
161+
assertEquals("null_column\r\n[]\r\n", text);
162+
}
163+
164+
public void testTsvMultiValueEmpty() {
165+
String text = TSV.format(req(), withData(singletonList(singletonList(emptyList()))));
166+
assertEquals("null_column\n[]\n", text);
167+
}
168+
169+
public void testCsvMultiValueNull() {
170+
String text = CSV.format(req(), withData(singletonList(singletonList(singletonList(null)))));
171+
assertEquals("null_column\r\n[NULL]\r\n", text);
172+
}
173+
174+
public void testTsvMultiValueNull() {
175+
String text = TSV.format(req(), withData(singletonList(singletonList(asList(null, null)))));
176+
assertEquals("null_column\n[NULL,NULL]\n", text);
177+
}
178+
179+
public void testCsvMultiValueKeywords() {
180+
String text = CSV.format(req(), withData(singletonList(singletonList(asList("one", "two", "one, two")))));
181+
assertEquals("keyword_column\r\n\"[\"\"one\"\",\"\"two\"\",\"\"one, two\"\"]\"\r\n", text);
182+
}
183+
184+
public void testTsvMultiValueKeywords() {
185+
String text = TSV.format(req(), withData(singletonList(singletonList(asList("one", "two", "one, two")))));
186+
assertEquals("keyword_column\n[\"one\",\"two\",\"one, two\"]\n", text);
187+
}
188+
189+
public void testCsvMultiValueKeywordWithQuote() {
190+
String text = CSV.format(req(), withData(singletonList(singletonList(asList("one", "two", "one\"two")))));
191+
assertEquals("keyword_column\r\n\"[\"\"one\"\",\"\"two\"\",\"\"one\\\"\"two\"\"]\"\r\n", text);
192+
}
193+
194+
public void testTsvMultiValueKeywordWithQuote() {
195+
String text = TSV.format(req(), withData(singletonList(singletonList(asList("one", "two", "one\"two")))));
196+
assertEquals("keyword_column\n[\"one\",\"two\",\"one\\\"two\"]\n", text);
197+
}
198+
199+
public void testTsvMultiValueKeywordWithTab() {
200+
String text = TSV.format(req(), withData(singletonList(singletonList(asList("one", "two", "one\ttwo")))));
201+
assertEquals("keyword_column\n[\"one\",\"two\",\"one\\ttwo\"]\n", text);
202+
}
203+
204+
public void testCsvMultiValueKeywordTwoColumns() {
205+
String text = CSV.format(req(), withData(singletonList(asList(asList("one", "two", "three"), asList("4", "5")))));
206+
assertEquals("keyword_column,keyword_column\r\n\"[\"\"one\"\",\"\"two\"\",\"\"three\"\"]\",\"[\"\"4\"\",\"\"5\"\"]\"\r\n", text);
207+
}
208+
209+
public void testTsvMultiValueKeywordTwoColumns() {
210+
String text = TSV.format(req(), withData(singletonList(asList(asList("one", "two", "three"), asList("4", "5")))));
211+
assertEquals("keyword_column\tkeyword_column\n[\"one\",\"two\",\"three\"]\t[\"4\",\"5\"]\n", text);
212+
}
213+
214+
public void testCsvMultiValueBooleans() {
215+
String text = CSV.format(req(), withData(singletonList(singletonList(asList(true, false, true)))));
216+
assertEquals("boolean_column\r\n\"[true,false,true]\"\r\n", text);
217+
}
218+
219+
public void testTsvMultiValueBooleans() {
220+
String text = TSV.format(req(), withData(singletonList(singletonList(asList(true, false, true)))));
221+
assertEquals("boolean_column\n[true,false,true]\n", text);
222+
}
223+
224+
public void testCsvMultiValueIntegers() {
225+
String text = CSV.format(req(), withData(singletonList(asList(
226+
asList((byte) 1, (byte) 2), asList((short) 3, (short) 4), asList(5, 6), asList(7L, 8L)
227+
))));
228+
assertEquals("byte_column,short_column,integer_column,long_column\r\n\"[1,2]\",\"[3,4]\",\"[5,6]\",\"[7,8]\"\r\n", text);
229+
}
230+
231+
public void testTsvMultiValueIntegers() {
232+
String text = TSV.format(req(), withData(singletonList(asList(
233+
asList((byte) 1, (byte) 2), asList((short) 3, (short) 4), asList(5, 6), asList(7L, 8L)
234+
))));
235+
assertEquals("byte_column\tshort_column\tinteger_column\tlong_column\n[1,2]\t[3,4]\t[5,6]\t[7,8]\n", text);
236+
}
237+
238+
public void testCsvMultiValueFloatingPoints() {
239+
String text = CSV.format(req(), withData(singletonList(asList(asList(1.1f, 2.2f), asList(3.3d, 4.4d)))));
240+
assertEquals("float_column,double_column\r\n\"[1.1,2.2]\",\"[3.3,4.4]\"\r\n", text);
241+
}
242+
243+
public void testTsvMultiValueFloatingPoints() {
244+
String text = TSV.format(req(), withData(singletonList(asList(asList(1.1f, 2.2f), asList(3.3d, 4.4d)))));
245+
assertEquals("float_column\tdouble_column\n[1.1,2.2]\t[3.3,4.4]\n", text);
246+
}
247+
248+
public void testCsvMultiValueDates() {
249+
String date1 = "2020-02-02T02:02:02.222+03:00";
250+
String date2 = "1969-01-23T23:34:56.123456789+13:30";
251+
String text = CSV.format(req(), withData(singletonList(singletonList(
252+
asList(ZonedDateTime.parse(date1), ZonedDateTime.parse(date2))
253+
))));
254+
assertEquals("datetime_column\r\n\"[" + date1 + "," + date2 + "]\"\r\n", text);
255+
}
256+
257+
public void testTsvMultiValueDates() {
258+
String date1 = "2020-02-02T02:02:02.222+03:00";
259+
String date2 = "1969-01-23T23:34:56.123456789+13:30";
260+
String text = TSV.format(req(), withData(singletonList(singletonList(
261+
asList(ZonedDateTime.parse(date1), ZonedDateTime.parse(date2))
262+
))));
263+
assertEquals("datetime_column\n[" + date1 + "," + date2 + "]\n", text);
264+
}
265+
266+
public void testCsvMultiValueGeoPoints() {
267+
GeoShape point1 = new GeoShape(12.34, 56.78);
268+
double lat = randomDouble(), lon = randomDouble();
269+
GeoShape point2 = new GeoShape(lat, lon);
270+
String text = CSV.format(req(), withData(singletonList(singletonList(asList(point1, point2)))));
271+
assertEquals("geo_shape_column\r\n\"[POINT (12.34 56.78),POINT (" + lat + " " + lon + ")]\"\r\n", text);
272+
}
273+
274+
public void testTsvMultiValueGeoPoints() {
275+
GeoShape point1 = new GeoShape(12.34, 56.78);
276+
double lat = randomDouble(), lon = randomDouble();
277+
GeoShape point2 = new GeoShape(lat, lon);
278+
String text = TSV.format(req(), withData(singletonList(singletonList(asList(point1, point2)))));
279+
assertEquals("geo_shape_column\n[POINT (12.34 56.78),POINT (" + lat + " " + lon + ")]\n", text);
280+
}
281+
282+
public void testCsvMultiValueSingletons() {
283+
String text = CSV.format(req(), withData(singletonList(asList(emptyList(), singletonList(false), singletonList("string"),
284+
singletonList(1), singletonList(2.), singletonList(new GeoShape(12.34, 56.78))))));
285+
assertEquals("null_column,boolean_column,keyword_column,integer_column,double_column,geo_shape_column\r\n" +
286+
"[],[false],\"[\"\"string\"\"]\",[1],[2.0],[POINT (12.34 56.78)]\r\n", text);
287+
}
288+
289+
public void testCsvMultiValueWithDelimiter() {
290+
String text = CSV.format(reqWithParam("delimiter", String.valueOf("|")),
291+
withData(singletonList(asList(emptyList(), asList(null, null), asList(false, true), asList("string", "strung"),
292+
asList(1, 2), asList(3.3, 4.), asList(new GeoShape(12.34, 56.78), new GeoShape(90, 10))))));
293+
assertEquals("null_column|null_column|boolean_column|keyword_column|integer_column|double_column|geo_shape_column\r\n" +
294+
"[]|[NULL,NULL]|[false,true]|\"[\"\"string\"\",\"\"strung\"\"]\"|[1,2]|[3.3,4.0]|[POINT (12.34 56.78),POINT (90.0 10.0)]\r\n",
295+
text);
296+
}
297+
298+
public void testTsvMultiValueSingletons() {
299+
String text = TSV.format(req(), withData(singletonList(asList(emptyList(), singletonList(false), singletonList("string"),
300+
singletonList(1), singletonList(2.), singletonList(new GeoShape(12.34, 56.78))))));
301+
assertEquals("null_column\tboolean_column\tkeyword_column\tinteger_column\tdouble_column\tgeo_shape_column\n" +
302+
"[]\t[false]\t[\"string\"]\t[1]\t[2.0]\t[POINT (12.34 56.78)]\n", text);
303+
}
155304

156305
private static SqlQueryResponse emptyData() {
157306
return new SqlQueryResponse(
@@ -164,6 +313,24 @@ private static SqlQueryResponse emptyData() {
164313
);
165314
}
166315

316+
private static SqlQueryResponse withData(List<List<Object>> rows) {
317+
List<ColumnInfo> headers = new ArrayList<>();
318+
if (rows.isEmpty() == false) {
319+
// headers
320+
for (Object o : rows.get(0)) {
321+
if (o instanceof Collection) {
322+
Collection<?> col = (Collection<?>) o;
323+
o = col.isEmpty() ? null : col.toArray()[0];
324+
}
325+
326+
String typeName = fromJava(o).typeName();
327+
headers.add(new ColumnInfo("index", typeName + "_column", typeName + "_array"));
328+
}
329+
}
330+
331+
return new SqlQueryResponse(null, Mode.JDBC, DATE_NANOS_SUPPORT_VERSION, false, headers, rows);
332+
}
333+
167334
private static SqlQueryResponse regularData() {
168335
// headers
169336
List<ColumnInfo> headers = new ArrayList<>();

0 commit comments

Comments
 (0)