From d25f77f4fb9c24bc49eb1357857d103ae9ba83f7 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 1 Oct 2018 12:18:30 -0700 Subject: [PATCH 1/7] When parsing JSON fields, also create tokens prefixed with the field key. --- .../index/mapper/JsonFieldMapper.java | 13 +- .../index/mapper/JsonFieldParser.java | 90 +++++++++++-- .../index/mapper/JsonFieldMapperTests.java | 59 ++++----- .../index/mapper/JsonFieldParserTests.java | 123 ++++++++++++++++-- 4 files changed, 230 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java index 7efcdf5ab9039..4caf7337b63e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java @@ -51,8 +51,10 @@ * of keys. * * Currently the mapper extracts all leaf values of the JSON object, converts them to their text - * representations, and indexes each one as a keyword. As an example, given a json field called - * 'json_field' and the following input + * representations, and indexes each one as a keyword. It creates both a prefixed version of the token + * to allow searches on particular key-value pairs, as well as a 'root' token that is not prefixed. + * + * As an example, given a json field called 'json_field' and the following input * * { * "json_field: { @@ -63,13 +65,18 @@ * } * } * - * the mapper will produce untokenized string fields with the values "some value" and "true". + * the mapper will produce untokenized string fields called "json_field" with values "some value" and "true", + * as well as string fields called "json_field._prefixed" with values "key\0some value" and "key2.key3\0true". + * + * Note that \0 is a reserved separator character, and cannot be used in the keys of the JSON object + * (see {@link JsonFieldParser#SEPARATOR}). */ public final class JsonFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "json"; public static final NamedAnalyzer WHITESPACE_ANALYZER = new NamedAnalyzer( "whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()); + public static final String PREFIXED_FIELD_SUFFIX = "._prefixed"; private static class Defaults { public static final MappedFieldType FIELD_TYPE = new JsonFieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index 25a40235844e9..fac9ee5ca2b0e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -31,16 +31,24 @@ /** * A helper class for {@link JsonFieldMapper} parses a JSON object - * and produces an indexable field for each leaf value. + * and produces a pair of indexable fields for each leaf value. */ public class JsonFieldParser { + private static final String SEPARATOR = "\0"; + private final MappedFieldType fieldType; private final int ignoreAbove; + private final String fieldName; + private final String prefixedFieldName; + JsonFieldParser(MappedFieldType fieldType, int ignoreAbove) { this.fieldType = fieldType; this.ignoreAbove = ignoreAbove; + + this.fieldName = fieldType.name(); + this.prefixedFieldName = fieldType.name() + JsonFieldMapper.PREFIXED_FIELD_SUFFIX; } public List parse(XContentParser parser) throws IOException { @@ -48,36 +56,92 @@ public List parse(XContentParser parser) throws IOException { parser.currentToken(), parser::getTokenLocation); + ContentPath path = new ContentPath(); List fields = new ArrayList<>(); - int openObjects = 1; + parseObject(parser, path, fields); + return fields; + } + + private void parseObject(XContentParser parser, + ContentPath path, + List fields) throws IOException { + String currentName = null; while (true) { - if (openObjects == 0) { - return fields; + XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.END_OBJECT) { + return; } + if (token == XContentParser.Token.FIELD_NAME) { + currentName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + assert currentName != null; + path.add(currentName); + parseObject(parser, path, fields); + path.remove(); + } else if (token == XContentParser.Token.START_ARRAY) { + assert currentName != null; + parseArray(parser, path, currentName, fields); + } else if (token.isValue()) { + String value = parser.text(); + addField(path, currentName, value, fields); + } else if (token == XContentParser.Token.VALUE_NULL) { + String value = fieldType.nullValueAsString(); + if (value != null) { + addField(path, currentName, value, fields); + } + } + } + } + + private void parseArray(XContentParser parser, + ContentPath path, + String currentName, + List fields) throws IOException { + while (true) { XContentParser.Token token = parser.nextToken(); - assert token != null; + if (token == XContentParser.Token.END_ARRAY) { + return; + } if (token == XContentParser.Token.START_OBJECT) { - openObjects++; - } else if (token == XContentParser.Token.END_OBJECT) { - openObjects--; + path.add(currentName); + parseObject(parser, path, fields); + path.remove(); } else if (token.isValue()) { String value = parser.text(); - addField(value, fields); + addField(path, currentName, value, fields); } else if (token == XContentParser.Token.VALUE_NULL) { String value = fieldType.nullValueAsString(); if (value != null) { - addField(value, fields); + addField(path, currentName, value, fields); } } } } - private void addField(String value, List fields) { - if (value.length() <= ignoreAbove) { - fields.add(new Field(fieldType.name(), new BytesRef(value), fieldType)); + private void addField(ContentPath path, + String currentName, + String value, + List fields) { + if (value.length() > ignoreAbove) { + return; + } + + assert currentName != null; + String key = path.pathAsText(currentName); + if (key.contains(SEPARATOR)) { + throw new IllegalArgumentException("Keys in [json] fields cannot contain the reserved character \\0." + + " Offending key: [" + key + "]."); } + String prefixedValue = createPrefixedValue(key, value); + + fields.add(new Field(fieldName, new BytesRef(value), fieldType)); + fields.add(new Field(prefixedFieldName, new BytesRef(prefixedValue), fieldType)); + } + + private static String createPrefixedValue(String key, String value) { + return key + SEPARATOR + value; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java index 91f73b4d61b53..db09f3046db1c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java @@ -72,30 +72,31 @@ public void testDefaults() throws Exception { BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() .startObject("field") - .field("key1", "value") - .field("key2", true) + .field("key", "value") .endObject() .endObject()); ParsedDocument parsedDoc = mapper.parse(SourceToParse.source("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); - assertEquals(2, fields.length); + assertEquals(1, fields.length); + + assertEquals("field", fields[0].name()); + assertEquals(new BytesRef("value"), fields[0].binaryValue()); + assertFalse(fields[0].fieldType().stored()); + assertTrue(fields[0].fieldType().omitNorms()); - IndexableField field1 = fields[0]; - assertEquals("field", field1.name()); - assertEquals(new BytesRef("value"), field1.binaryValue()); - assertTrue(field1.fieldType().omitNorms()); + IndexableField[] prefixedFields = parsedDoc.rootDoc().getFields("field._prefixed"); + assertEquals(1, prefixedFields.length); - IndexableField field2 = fields[1]; - assertEquals("field", field2.name()); - assertEquals(new BytesRef("true"), field2.binaryValue()); - assertTrue(field2.fieldType().omitNorms()); + assertEquals("field._prefixed", prefixedFields[0].name()); + assertEquals(new BytesRef("key\0value"), prefixedFields[0].binaryValue()); + assertFalse(prefixedFields[0].fieldType().stored()); + assertTrue(prefixedFields[0].fieldType().omitNorms()); IndexableField[] fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME); assertEquals(1, fieldNamesFields.length); - - IndexableField fieldNamesField = fieldNamesFields[0]; - assertEquals("field", fieldNamesField.stringValue()); + assertEquals("field", fieldNamesFields[0].stringValue()); } public void testDisableIndex() throws Exception { @@ -248,20 +249,18 @@ public void testFieldMultiplicity() throws Exception { .endObject()); ParsedDocument parsedDoc = mapper.parse(SourceToParse.source("test", "type", "1", doc, XContentType.JSON)); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); assertEquals(3, fields.length); - - IndexableField field1 = fields[0]; - assertEquals("field", field1.name()); - assertEquals(new BytesRef("value"), field1.binaryValue()); - - IndexableField field2 = fields[1]; - assertEquals("field", field2.name()); - assertEquals(new BytesRef("true"), field2.binaryValue()); - - IndexableField field3 = fields[2]; - assertEquals("field", field3.name()); - assertEquals(new BytesRef("false"), field3.binaryValue()); + assertEquals(new BytesRef("value"), fields[0].binaryValue()); + assertEquals(new BytesRef("true"), fields[1].binaryValue()); + assertEquals(new BytesRef("false"), fields[2].binaryValue()); + + IndexableField[] prefixedFields = parsedDoc.rootDoc().getFields("field._prefixed"); + assertEquals(3, prefixedFields.length); + assertEquals(new BytesRef("key1\0value"), prefixedFields[0].binaryValue()); + assertEquals(new BytesRef("key2\0true"), prefixedFields[1].binaryValue()); + assertEquals(new BytesRef("key3\0false"), prefixedFields[2].binaryValue()); } public void testIgnoreAbove() throws IOException { @@ -292,7 +291,6 @@ public void testIgnoreAbove() throws IOException { assertEquals(0, fields.length); } - public void testNullValues() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() .startObject("type") @@ -326,8 +324,11 @@ public void testNullValues() throws Exception { IndexableField[] otherFields = parsedDoc.rootDoc().getFields("other_field"); assertEquals(1, otherFields.length); - IndexableField field = otherFields[0]; - assertEquals(new BytesRef("placeholder"), field.binaryValue()); + assertEquals(new BytesRef("placeholder"), otherFields[0].binaryValue()); + + IndexableField[] prefixedOtherFields = parsedDoc.rootDoc().getFields("other_field._prefixed"); + assertEquals(1, prefixedOtherFields.length); + assertEquals(new BytesRef("key\0placeholder"), prefixedOtherFields[0].binaryValue()); } public void testSplitQueriesOnWhitespace() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index a60637a87e14b..4ed265ef6b86d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -19,8 +19,10 @@ package org.elasticsearch.index.mapper; +import com.fasterxml.jackson.core.JsonParseException; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.JsonFieldMapper.JsonFieldType; @@ -47,15 +49,23 @@ public void testTextValues() throws Exception { XContentParser xContentParser = createXContentParser(input); List fields = parser.parse(xContentParser); - assertEquals(2, fields.size()); + assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); assertEquals("field", field1.name()); assertEquals(new BytesRef("value1"), field1.binaryValue()); - IndexableField field2 = fields.get(1); + IndexableField prefixedField1 = fields.get(1); + assertEquals("field._prefixed", prefixedField1.name()); + assertEquals(new BytesRef("key1\0value1"), prefixedField1.binaryValue()); + + IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("value2"), field2.binaryValue()); + + IndexableField prefixedField2 = fields.get(3); + assertEquals("field._prefixed", prefixedField2.name()); + assertEquals(new BytesRef("key2\0value2"), prefixedField2.binaryValue()); } public void testNumericValues() throws Exception { @@ -63,11 +73,15 @@ public void testNumericValues() throws Exception { XContentParser xContentParser = createXContentParser(input); List fields = parser.parse(xContentParser); - assertEquals(1, fields.size()); + assertEquals(2, fields.size()); IndexableField field = fields.get(0); assertEquals("field", field.name()); assertEquals(new BytesRef("2.718"), field.binaryValue()); + + IndexableField prefixedField = fields.get(1); + assertEquals("field._prefixed", prefixedField.name()); + assertEquals(new BytesRef("key" + '\0' + "2.718"), prefixedField.binaryValue()); } public void testBooleanValues() throws Exception { @@ -75,27 +89,71 @@ public void testBooleanValues() throws Exception { XContentParser xContentParser = createXContentParser(input); List fields = parser.parse(xContentParser); - assertEquals(1, fields.size()); + assertEquals(2, fields.size()); IndexableField field = fields.get(0); assertEquals("field", field.name()); assertEquals(new BytesRef("false"), field.binaryValue()); + + IndexableField prefixedField = fields.get(1); + assertEquals("field._prefixed", prefixedField.name()); + assertEquals(new BytesRef("key\0false"), prefixedField.binaryValue()); } - public void testArrays() throws Exception { + public void testBasicArrays() throws Exception { String input = "{ \"key\": [true, false] }"; XContentParser xContentParser = createXContentParser(input); List fields = parser.parse(xContentParser); - assertEquals(2, fields.size()); + assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); assertEquals("field", field1.name()); assertEquals(new BytesRef("true"), field1.binaryValue()); - IndexableField field2 = fields.get(1); + IndexableField prefixedField1 = fields.get(1); + assertEquals("field._prefixed", prefixedField1.name()); + assertEquals(new BytesRef("key\0true"), prefixedField1.binaryValue()); + + IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("false"), field2.binaryValue()); + + IndexableField prefixedField2 = fields.get(3); + assertEquals("field._prefixed", prefixedField2.name()); + assertEquals(new BytesRef("key\0false"), prefixedField2.binaryValue()); + } + + public void testArraysOfObjects() throws Exception { + String input = "{ \"key1\": [{ \"key2\": true }, false], \"key4\": \"other\" }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(6, fields.size()); + + IndexableField field1 = fields.get(0); + assertEquals("field", field1.name()); + assertEquals(new BytesRef("true"), field1.binaryValue()); + + IndexableField prefixedField1 = fields.get(1); + assertEquals("field._prefixed", prefixedField1.name()); + assertEquals(new BytesRef("key1.key2\0true"), prefixedField1.binaryValue()); + + IndexableField field2 = fields.get(2); + assertEquals("field", field2.name()); + assertEquals(new BytesRef("false"), field2.binaryValue()); + + IndexableField prefixedField2 = fields.get(3); + assertEquals("field._prefixed", prefixedField2.name()); + assertEquals(new BytesRef("key1\0false"), prefixedField2.binaryValue()); + + IndexableField field3 = fields.get(4); + assertEquals("field", field3.name()); + assertEquals(new BytesRef("other"), field3.binaryValue()); + + IndexableField prefixedField3 = fields.get(5); + assertEquals("field._prefixed", prefixedField3.name()); + assertEquals(new BytesRef("key4\0other"), prefixedField3.binaryValue()); } public void testNestedObjects() throws Exception { @@ -104,15 +162,23 @@ public void testNestedObjects() throws Exception { XContentParser xContentParser = createXContentParser(input); List fields = parser.parse(xContentParser); - assertEquals(2, fields.size()); + assertEquals(4, fields.size()); IndexableField field1 = fields.get(0); assertEquals("field", field1.name()); assertEquals(new BytesRef("value"), field1.binaryValue()); - IndexableField field2 = fields.get(1); + IndexableField prefixedField1 = fields.get(1); + assertEquals("field._prefixed", prefixedField1.name()); + assertEquals(new BytesRef("parent1.key\0value"), prefixedField1.binaryValue()); + + IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("value"), field2.binaryValue()); + + IndexableField prefixedField2 = fields.get(3); + assertEquals("field._prefixed", prefixedField2.name()); + assertEquals(new BytesRef("parent2.key\0value"), prefixedField2.binaryValue()); } public void testIgnoreAbove() throws Exception { @@ -142,11 +208,42 @@ public void testNullValues() throws Exception { JsonFieldParser nullValueParser = new JsonFieldParser(fieldType, Integer.MAX_VALUE); fields = nullValueParser.parse(xContentParser); - assertEquals(1, fields.size()); + assertEquals(2, fields.size()); IndexableField field = fields.get(0); assertEquals("field", field.name()); assertEquals(new BytesRef("placeholder"), field.binaryValue()); + + IndexableField prefixedField = fields.get(1); + assertEquals("field._prefixed", prefixedField.name()); + assertEquals(new BytesRef("key\0placeholder"), prefixedField.binaryValue()); + } + + public void testMalformedJson() throws Exception { + String input = "{ \"key\": [true, false }"; + XContentParser xContentParser = createXContentParser(input); + + expectThrows(JsonParseException.class, () -> parser.parse(xContentParser)); + } + + public void testEmptyObject() throws Exception { + String input = "{}"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(0, fields.size()); + } + + public void testReservedCharacters() throws Exception { + XContentBuilder input = XContentBuilder.builder(JsonXContent.jsonXContent) + .startObject() + .field("k\0y", "value") + .endObject(); + XContentParser xContentParser = createXContentParser(input); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parser.parse(xContentParser)); + assertEquals("Keys in [json] fields cannot contain the reserved character \\0. Offending key: [k\0y].", + e.getMessage()); } private XContentParser createXContentParser(String input) throws IOException { @@ -154,4 +251,10 @@ private XContentParser createXContentParser(String input) throws IOException { xContentParser.nextToken(); return xContentParser; } + + private XContentParser createXContentParser(XContentBuilder input) throws IOException { + XContentParser xContentParser = createParser(input); + xContentParser.nextToken(); + return xContentParser; + } } From fce305d48206e841f6ceab3858084460b84756e6 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 1 Oct 2018 13:29:12 -0700 Subject: [PATCH 2/7] Add an integration test for search queries. --- .../search/query/SearchQueryIT.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 6068f89025994..34bec5e1211ce 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1780,4 +1780,45 @@ public void testFieldAliasesForMetaFields() throws Exception { DocumentField field = hit.getFields().get("id-alias"); assertThat(field.getValue().toString(), equalTo("1")); } + + public void testJsonField() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("properties") + .startObject("headers") + .field("type", "json") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(prepareCreate("test").addMapping("type", mapping)); + + XContentBuilder source = XContentFactory.jsonBuilder() + .startObject() + .startObject("headers") + .field("content-type", "application/json") + .endObject() + .endObject(); + IndexRequestBuilder indexRequest = client().prepareIndex("test", "type") + .setId("1") + .setRouting("custom") + .setSource(source); + indexRandom(true, false, indexRequest); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(prefixQuery("headers", "application/")) + .get(); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch() + .setQuery(existsQuery("headers")) + .get(); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch() + .setQuery(prefixQuery("headers", "content")) + .get(); + assertHitCount(searchResponse, 0L); + } } From 61bd1671c1bb725d37e9d4c70f4fcc27e1a41ab3 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 2 Oct 2018 11:03:26 -0700 Subject: [PATCH 3/7] In JsonFieldParser, rename fieldName -> rootFieldName. --- .../org/elasticsearch/index/mapper/JsonFieldParser.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index fac9ee5ca2b0e..de89ac7cfeefd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -39,7 +39,7 @@ public class JsonFieldParser { private final MappedFieldType fieldType; private final int ignoreAbove; - private final String fieldName; + private final String rootFieldName; private final String prefixedFieldName; JsonFieldParser(MappedFieldType fieldType, @@ -47,7 +47,7 @@ public class JsonFieldParser { this.fieldType = fieldType; this.ignoreAbove = ignoreAbove; - this.fieldName = fieldType.name(); + this.rootFieldName = fieldType.name(); this.prefixedFieldName = fieldType.name() + JsonFieldMapper.PREFIXED_FIELD_SUFFIX; } @@ -136,8 +136,8 @@ private void addField(ContentPath path, + " Offending key: [" + key + "]."); } String prefixedValue = createPrefixedValue(key, value); - - fields.add(new Field(fieldName, new BytesRef(value), fieldType)); + + fields.add(new Field(rootFieldName, new BytesRef(value), fieldType)); fields.add(new Field(prefixedFieldName, new BytesRef(prefixedValue), fieldType)); } From 8facb517eb229b8c32479e53bb2fe335a90e932e Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 2 Oct 2018 13:41:38 -0700 Subject: [PATCH 4/7] Make sure that arrays of arrays are supported. --- .../index/mapper/JsonFieldParser.java | 55 +++++++++---------- .../index/mapper/JsonFieldParserTests.java | 32 +++++++++++ 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index de89ac7cfeefd..7435094d98959 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -75,48 +75,44 @@ private void parseObject(XContentParser parser, if (token == XContentParser.Token.FIELD_NAME) { currentName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { + } else { assert currentName != null; - path.add(currentName); - parseObject(parser, path, fields); - path.remove(); - } else if (token == XContentParser.Token.START_ARRAY) { - assert currentName != null; - parseArray(parser, path, currentName, fields); - } else if (token.isValue()) { - String value = parser.text(); - addField(path, currentName, value, fields); - } else if (token == XContentParser.Token.VALUE_NULL) { - String value = fieldType.nullValueAsString(); - if (value != null) { - addField(path, currentName, value, fields); - } + parseFieldValue(token, parser, path, currentName, fields); } } } private void parseArray(XContentParser parser, - ContentPath path, - String currentName, - List fields) throws IOException { + ContentPath path, + String currentName, + List fields) throws IOException { while (true) { XContentParser.Token token = parser.nextToken(); if (token == XContentParser.Token.END_ARRAY) { return; } + parseFieldValue(token, parser, path, currentName, fields); + } + } - if (token == XContentParser.Token.START_OBJECT) { - path.add(currentName); - parseObject(parser, path, fields); - path.remove(); - } else if (token.isValue()) { - String value = parser.text(); + private void parseFieldValue(XContentParser.Token token, + XContentParser parser, + ContentPath path, + String currentName, + List fields) throws IOException { + if (token == XContentParser.Token.START_OBJECT) { + path.add(currentName); + parseObject(parser, path, fields); + path.remove(); + } else if (token == XContentParser.Token.START_ARRAY) { + parseArray(parser, path, currentName, fields); + } else if (token.isValue()) { + String value = parser.text(); + addField(path, currentName, value, fields); + } else if (token == XContentParser.Token.VALUE_NULL) { + String value = fieldType.nullValueAsString(); + if (value != null) { addField(path, currentName, value, fields); - } else if (token == XContentParser.Token.VALUE_NULL) { - String value = fieldType.nullValueAsString(); - if (value != null) { - addField(path, currentName, value, fields); - } } } } @@ -129,7 +125,6 @@ private void addField(ContentPath path, return; } - assert currentName != null; String key = path.pathAsText(currentName); if (key.contains(SEPARATOR)) { throw new IllegalArgumentException("Keys in [json] fields cannot contain the reserved character \\0." diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index 4ed265ef6b86d..f035c45bff25d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -124,6 +124,38 @@ public void testBasicArrays() throws Exception { assertEquals(new BytesRef("key\0false"), prefixedField2.binaryValue()); } + public void testArrayOfArrays() throws Exception { + String input = "{ \"key\": [[true, \"value\"], 3] }"; + XContentParser xContentParser = createXContentParser(input); + + List fields = parser.parse(xContentParser); + assertEquals(6, fields.size()); + + IndexableField field1 = fields.get(0); + assertEquals("field", field1.name()); + assertEquals(new BytesRef("true"), field1.binaryValue()); + + IndexableField prefixedField1 = fields.get(1); + assertEquals("field._prefixed", prefixedField1.name()); + assertEquals(new BytesRef("key\0true"), prefixedField1.binaryValue()); + + IndexableField field2 = fields.get(2); + assertEquals("field", field2.name()); + assertEquals(new BytesRef("value"), field2.binaryValue()); + + IndexableField prefixedField2 = fields.get(3); + assertEquals("field._prefixed", prefixedField2.name()); + assertEquals(new BytesRef("key\0value"), prefixedField2.binaryValue()); + + IndexableField field3 = fields.get(4); + assertEquals("field", field3.name()); + assertEquals(new BytesRef("3"), field3.binaryValue()); + + IndexableField prefixedField3 = fields.get(5); + assertEquals("field._prefixed", prefixedField3.name()); + assertEquals(new BytesRef("key" + "\0" + "3"), prefixedField3.binaryValue()); + } + public void testArraysOfObjects() throws Exception { String input = "{ \"key1\": [{ \"key2\": true }, false], \"key4\": \"other\" }"; XContentParser xContentParser = createXContentParser(input); From 6876518c959d324dd8e3140de226e9899537cbdb Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 2 Oct 2018 14:03:57 -0700 Subject: [PATCH 5/7] When parsing JSON, fail hard when encountering an unexpected token. --- .../java/org/elasticsearch/index/mapper/JsonFieldParser.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index 7435094d98959..5dfbdb56c73fd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -114,6 +114,10 @@ private void parseFieldValue(XContentParser.Token token, if (value != null) { addField(path, currentName, value, fields); } + } else { + // Note that we throw an exception here just to be safe. We don't actually expect to reach + // this case, since XContentParser verifies that the input is well-formed as it parses. + throw new IllegalArgumentException("Encountered unexpected token [" + token.toString() + "]."); } } From 66279325b87240d1ae2cd231b9099377aa899d31 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 2 Oct 2018 14:04:34 -0700 Subject: [PATCH 6/7] Add a randomized test to help ensure JSON parsing is robust. --- .../index/mapper/JsonFieldParserTests.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index f035c45bff25d..f69847c0d200d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -22,11 +22,14 @@ import com.fasterxml.jackson.core.JsonParseException; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.JsonFieldMapper.JsonFieldType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.XContentTestUtils; import org.junit.Before; import java.io.IOException; @@ -266,12 +269,32 @@ public void testEmptyObject() throws Exception { assertEquals(0, fields.size()); } + public void testRandomFields() throws Exception { + BytesReference input = BytesReference.bytes( + XContentBuilder.builder(JsonXContent.jsonXContent) + .startObject() + .startObject("object") + .field("key", "value") + .endObject() + .startArray("array") + .value(2.718) + .endArray() + .endObject()); + + input = XContentTestUtils.insertRandomFields(XContentType.JSON, input, null, random()); + XContentParser xContentParser = createXContentParser(input.utf8ToString()); + + List fields = parser.parse(xContentParser); + assertTrue(fields.size() > 4); + } + public void testReservedCharacters() throws Exception { - XContentBuilder input = XContentBuilder.builder(JsonXContent.jsonXContent) - .startObject() - .field("k\0y", "value") - .endObject(); - XContentParser xContentParser = createXContentParser(input); + BytesReference input = BytesReference.bytes( + XContentBuilder.builder(JsonXContent.jsonXContent) + .startObject() + .field("k\0y", "value") + .endObject()); + XContentParser xContentParser = createXContentParser(input.utf8ToString()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parser.parse(xContentParser)); assertEquals("Keys in [json] fields cannot contain the reserved character \\0. Offending key: [k\0y].", @@ -283,10 +306,4 @@ private XContentParser createXContentParser(String input) throws IOException { xContentParser.nextToken(); return xContentParser; } - - private XContentParser createXContentParser(XContentBuilder input) throws IOException { - XContentParser xContentParser = createParser(input); - xContentParser.nextToken(); - return xContentParser; - } } From 21534d9b3f503acb74352b68126727b43b566aa4 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 2 Oct 2018 14:07:28 -0700 Subject: [PATCH 7/7] Use the term 'keyed' as opposed to 'prefixed'. --- .../index/mapper/JsonFieldMapper.java | 8 +- .../index/mapper/JsonFieldParser.java | 10 +-- .../index/mapper/JsonFieldMapperTests.java | 24 ++--- .../index/mapper/JsonFieldParserTests.java | 90 +++++++++---------- 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java index 4caf7337b63e6..bc6bcec26008d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java @@ -51,8 +51,8 @@ * of keys. * * Currently the mapper extracts all leaf values of the JSON object, converts them to their text - * representations, and indexes each one as a keyword. It creates both a prefixed version of the token - * to allow searches on particular key-value pairs, as well as a 'root' token that is not prefixed. + * representations, and indexes each one as a keyword. It creates both a 'keyed' version of the token + * to allow searches on particular key-value pairs, as well as a 'root' token without the key * * As an example, given a json field called 'json_field' and the following input * @@ -66,7 +66,7 @@ * } * * the mapper will produce untokenized string fields called "json_field" with values "some value" and "true", - * as well as string fields called "json_field._prefixed" with values "key\0some value" and "key2.key3\0true". + * as well as string fields called "json_field._keyed" with values "key\0some value" and "key2.key3\0true". * * Note that \0 is a reserved separator character, and cannot be used in the keys of the JSON object * (see {@link JsonFieldParser#SEPARATOR}). @@ -76,7 +76,7 @@ public final class JsonFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "json"; public static final NamedAnalyzer WHITESPACE_ANALYZER = new NamedAnalyzer( "whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()); - public static final String PREFIXED_FIELD_SUFFIX = "._prefixed"; + public static final String KEYED_FIELD_SUFFIX = "._keyed"; private static class Defaults { public static final MappedFieldType FIELD_TYPE = new JsonFieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index 5dfbdb56c73fd..9a1ed8d4d9bf1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -40,7 +40,7 @@ public class JsonFieldParser { private final int ignoreAbove; private final String rootFieldName; - private final String prefixedFieldName; + private final String keyedFieldName; JsonFieldParser(MappedFieldType fieldType, int ignoreAbove) { @@ -48,7 +48,7 @@ public class JsonFieldParser { this.ignoreAbove = ignoreAbove; this.rootFieldName = fieldType.name(); - this.prefixedFieldName = fieldType.name() + JsonFieldMapper.PREFIXED_FIELD_SUFFIX; + this.keyedFieldName = fieldType.name() + JsonFieldMapper.KEYED_FIELD_SUFFIX; } public List parse(XContentParser parser) throws IOException { @@ -134,13 +134,13 @@ private void addField(ContentPath path, throw new IllegalArgumentException("Keys in [json] fields cannot contain the reserved character \\0." + " Offending key: [" + key + "]."); } - String prefixedValue = createPrefixedValue(key, value); + String keyedValue = createKeyedValue(key, value); fields.add(new Field(rootFieldName, new BytesRef(value), fieldType)); - fields.add(new Field(prefixedFieldName, new BytesRef(prefixedValue), fieldType)); + fields.add(new Field(keyedFieldName, new BytesRef(keyedValue), fieldType)); } - private static String createPrefixedValue(String key, String value) { + private static String createKeyedValue(String key, String value) { return key + SEPARATOR + value; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java index db09f3046db1c..50e471be66391 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java @@ -86,13 +86,13 @@ public void testDefaults() throws Exception { assertFalse(fields[0].fieldType().stored()); assertTrue(fields[0].fieldType().omitNorms()); - IndexableField[] prefixedFields = parsedDoc.rootDoc().getFields("field._prefixed"); - assertEquals(1, prefixedFields.length); + IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed"); + assertEquals(1, keyedFields.length); - assertEquals("field._prefixed", prefixedFields[0].name()); - assertEquals(new BytesRef("key\0value"), prefixedFields[0].binaryValue()); - assertFalse(prefixedFields[0].fieldType().stored()); - assertTrue(prefixedFields[0].fieldType().omitNorms()); + assertEquals("field._keyed", keyedFields[0].name()); + assertEquals(new BytesRef("key\0value"), keyedFields[0].binaryValue()); + assertFalse(keyedFields[0].fieldType().stored()); + assertTrue(keyedFields[0].fieldType().omitNorms()); IndexableField[] fieldNamesFields = parsedDoc.rootDoc().getFields(FieldNamesFieldMapper.NAME); assertEquals(1, fieldNamesFields.length); @@ -256,11 +256,11 @@ public void testFieldMultiplicity() throws Exception { assertEquals(new BytesRef("true"), fields[1].binaryValue()); assertEquals(new BytesRef("false"), fields[2].binaryValue()); - IndexableField[] prefixedFields = parsedDoc.rootDoc().getFields("field._prefixed"); - assertEquals(3, prefixedFields.length); - assertEquals(new BytesRef("key1\0value"), prefixedFields[0].binaryValue()); - assertEquals(new BytesRef("key2\0true"), prefixedFields[1].binaryValue()); - assertEquals(new BytesRef("key3\0false"), prefixedFields[2].binaryValue()); + IndexableField[] keyedFields = parsedDoc.rootDoc().getFields("field._keyed"); + assertEquals(3, keyedFields.length); + assertEquals(new BytesRef("key1\0value"), keyedFields[0].binaryValue()); + assertEquals(new BytesRef("key2\0true"), keyedFields[1].binaryValue()); + assertEquals(new BytesRef("key3\0false"), keyedFields[2].binaryValue()); } public void testIgnoreAbove() throws IOException { @@ -326,7 +326,7 @@ public void testNullValues() throws Exception { assertEquals(1, otherFields.length); assertEquals(new BytesRef("placeholder"), otherFields[0].binaryValue()); - IndexableField[] prefixedOtherFields = parsedDoc.rootDoc().getFields("other_field._prefixed"); + IndexableField[] prefixedOtherFields = parsedDoc.rootDoc().getFields("other_field._keyed"); assertEquals(1, prefixedOtherFields.length); assertEquals(new BytesRef("key\0placeholder"), prefixedOtherFields[0].binaryValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index f69847c0d200d..a39513a0ac319 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -58,17 +58,17 @@ public void testTextValues() throws Exception { assertEquals("field", field1.name()); assertEquals(new BytesRef("value1"), field1.binaryValue()); - IndexableField prefixedField1 = fields.get(1); - assertEquals("field._prefixed", prefixedField1.name()); - assertEquals(new BytesRef("key1\0value1"), prefixedField1.binaryValue()); + IndexableField keyedField1 = fields.get(1); + assertEquals("field._keyed", keyedField1.name()); + assertEquals(new BytesRef("key1\0value1"), keyedField1.binaryValue()); IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("value2"), field2.binaryValue()); - IndexableField prefixedField2 = fields.get(3); - assertEquals("field._prefixed", prefixedField2.name()); - assertEquals(new BytesRef("key2\0value2"), prefixedField2.binaryValue()); + IndexableField keyedField2 = fields.get(3); + assertEquals("field._keyed", keyedField2.name()); + assertEquals(new BytesRef("key2\0value2"), keyedField2.binaryValue()); } public void testNumericValues() throws Exception { @@ -82,9 +82,9 @@ public void testNumericValues() throws Exception { assertEquals("field", field.name()); assertEquals(new BytesRef("2.718"), field.binaryValue()); - IndexableField prefixedField = fields.get(1); - assertEquals("field._prefixed", prefixedField.name()); - assertEquals(new BytesRef("key" + '\0' + "2.718"), prefixedField.binaryValue()); + IndexableField keyedField = fields.get(1); + assertEquals("field._keyed", keyedField.name()); + assertEquals(new BytesRef("key" + '\0' + "2.718"), keyedField.binaryValue()); } public void testBooleanValues() throws Exception { @@ -98,9 +98,9 @@ public void testBooleanValues() throws Exception { assertEquals("field", field.name()); assertEquals(new BytesRef("false"), field.binaryValue()); - IndexableField prefixedField = fields.get(1); - assertEquals("field._prefixed", prefixedField.name()); - assertEquals(new BytesRef("key\0false"), prefixedField.binaryValue()); + IndexableField keyedField = fields.get(1); + assertEquals("field._keyed", keyedField.name()); + assertEquals(new BytesRef("key\0false"), keyedField.binaryValue()); } public void testBasicArrays() throws Exception { @@ -114,17 +114,17 @@ public void testBasicArrays() throws Exception { assertEquals("field", field1.name()); assertEquals(new BytesRef("true"), field1.binaryValue()); - IndexableField prefixedField1 = fields.get(1); - assertEquals("field._prefixed", prefixedField1.name()); - assertEquals(new BytesRef("key\0true"), prefixedField1.binaryValue()); + IndexableField keyedField1 = fields.get(1); + assertEquals("field._keyed", keyedField1.name()); + assertEquals(new BytesRef("key\0true"), keyedField1.binaryValue()); IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("false"), field2.binaryValue()); - IndexableField prefixedField2 = fields.get(3); - assertEquals("field._prefixed", prefixedField2.name()); - assertEquals(new BytesRef("key\0false"), prefixedField2.binaryValue()); + IndexableField keyedField2 = fields.get(3); + assertEquals("field._keyed", keyedField2.name()); + assertEquals(new BytesRef("key\0false"), keyedField2.binaryValue()); } public void testArrayOfArrays() throws Exception { @@ -138,25 +138,25 @@ public void testArrayOfArrays() throws Exception { assertEquals("field", field1.name()); assertEquals(new BytesRef("true"), field1.binaryValue()); - IndexableField prefixedField1 = fields.get(1); - assertEquals("field._prefixed", prefixedField1.name()); - assertEquals(new BytesRef("key\0true"), prefixedField1.binaryValue()); + IndexableField keyedField1 = fields.get(1); + assertEquals("field._keyed", keyedField1.name()); + assertEquals(new BytesRef("key\0true"), keyedField1.binaryValue()); IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("value"), field2.binaryValue()); - IndexableField prefixedField2 = fields.get(3); - assertEquals("field._prefixed", prefixedField2.name()); - assertEquals(new BytesRef("key\0value"), prefixedField2.binaryValue()); + IndexableField keyedField2 = fields.get(3); + assertEquals("field._keyed", keyedField2.name()); + assertEquals(new BytesRef("key\0value"), keyedField2.binaryValue()); IndexableField field3 = fields.get(4); assertEquals("field", field3.name()); assertEquals(new BytesRef("3"), field3.binaryValue()); - IndexableField prefixedField3 = fields.get(5); - assertEquals("field._prefixed", prefixedField3.name()); - assertEquals(new BytesRef("key" + "\0" + "3"), prefixedField3.binaryValue()); + IndexableField keyedField3 = fields.get(5); + assertEquals("field._keyed", keyedField3.name()); + assertEquals(new BytesRef("key" + "\0" + "3"), keyedField3.binaryValue()); } public void testArraysOfObjects() throws Exception { @@ -170,25 +170,25 @@ public void testArraysOfObjects() throws Exception { assertEquals("field", field1.name()); assertEquals(new BytesRef("true"), field1.binaryValue()); - IndexableField prefixedField1 = fields.get(1); - assertEquals("field._prefixed", prefixedField1.name()); - assertEquals(new BytesRef("key1.key2\0true"), prefixedField1.binaryValue()); + IndexableField keyedField1 = fields.get(1); + assertEquals("field._keyed", keyedField1.name()); + assertEquals(new BytesRef("key1.key2\0true"), keyedField1.binaryValue()); IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("false"), field2.binaryValue()); - IndexableField prefixedField2 = fields.get(3); - assertEquals("field._prefixed", prefixedField2.name()); - assertEquals(new BytesRef("key1\0false"), prefixedField2.binaryValue()); + IndexableField keyedField2 = fields.get(3); + assertEquals("field._keyed", keyedField2.name()); + assertEquals(new BytesRef("key1\0false"), keyedField2.binaryValue()); IndexableField field3 = fields.get(4); assertEquals("field", field3.name()); assertEquals(new BytesRef("other"), field3.binaryValue()); - IndexableField prefixedField3 = fields.get(5); - assertEquals("field._prefixed", prefixedField3.name()); - assertEquals(new BytesRef("key4\0other"), prefixedField3.binaryValue()); + IndexableField keyedField3 = fields.get(5); + assertEquals("field._keyed", keyedField3.name()); + assertEquals(new BytesRef("key4\0other"), keyedField3.binaryValue()); } public void testNestedObjects() throws Exception { @@ -203,17 +203,17 @@ public void testNestedObjects() throws Exception { assertEquals("field", field1.name()); assertEquals(new BytesRef("value"), field1.binaryValue()); - IndexableField prefixedField1 = fields.get(1); - assertEquals("field._prefixed", prefixedField1.name()); - assertEquals(new BytesRef("parent1.key\0value"), prefixedField1.binaryValue()); + IndexableField keyedField1 = fields.get(1); + assertEquals("field._keyed", keyedField1.name()); + assertEquals(new BytesRef("parent1.key\0value"), keyedField1.binaryValue()); IndexableField field2 = fields.get(2); assertEquals("field", field2.name()); assertEquals(new BytesRef("value"), field2.binaryValue()); - IndexableField prefixedField2 = fields.get(3); - assertEquals("field._prefixed", prefixedField2.name()); - assertEquals(new BytesRef("parent2.key\0value"), prefixedField2.binaryValue()); + IndexableField keyedField2 = fields.get(3); + assertEquals("field._keyed", keyedField2.name()); + assertEquals(new BytesRef("parent2.key\0value"), keyedField2.binaryValue()); } public void testIgnoreAbove() throws Exception { @@ -249,9 +249,9 @@ public void testNullValues() throws Exception { assertEquals("field", field.name()); assertEquals(new BytesRef("placeholder"), field.binaryValue()); - IndexableField prefixedField = fields.get(1); - assertEquals("field._prefixed", prefixedField.name()); - assertEquals(new BytesRef("key\0placeholder"), prefixedField.binaryValue()); + IndexableField keyedField = fields.get(1); + assertEquals("field._keyed", keyedField.name()); + assertEquals(new BytesRef("key\0placeholder"), keyedField.binaryValue()); } public void testMalformedJson() throws Exception {