From e2f73bb1064cf4ae23a92741147d07826f4c2ea4 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 20 Jun 2018 10:14:07 +0100 Subject: [PATCH 01/17] Non-detailed case --- .../elasticsearch/client/IndicesClient.java | 13 +++ .../client/RequestConverters.java | 12 +++ .../elasticsearch/client/IndicesClientIT.java | 13 +++ .../admin/indices/analyze/AnalyzeRequest.java | 51 ++++++++- .../indices/analyze/AnalyzeResponse.java | 101 ++++++++++++++++++ .../indices/analyze/AnalyzeResponseTests.java | 54 ++++++++++ 6 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 30a42eb333f4a..4f0df184cc82f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -721,4 +723,15 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, RequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); } + + public AnalyzeResponse analyze(AnalyzeRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::analyze, options, + AnalyzeResponse::fromXContent, emptySet()); + } + + public void analyzeAsync(AnalyzeRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::analyze, options, + AnalyzeResponse::fromXContent, listener, emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index ab85af9f1fd7e..826d2d7846d9e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -892,6 +893,17 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) t return request; } + static Request analyze(AnalyzeRequest request) throws IOException { + EndpointBuilder builder = new EndpointBuilder().addPathPartAsIs("_analyze"); + String index = request.index(); + if (index != null) { + builder.addPathPart(index); + } + Request req = new Request(HttpGet.METHOD_NAME, builder.build()); + req.setEntity(createEntity(request, REQUEST_BODY_CONTENT_TYPE)); + return req; + } + private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef(); return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index c226b5349267c..2778c6ea9a168 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -29,6 +29,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -1240,4 +1242,15 @@ public void testGetIndexTemplate() throws Exception { new GetIndexTemplatesRequest().names("the-template-*"), client.indices()::getTemplate, client.indices()::getTemplateAsync)); assertThat(notFound.status(), equalTo(RestStatus.NOT_FOUND)); } + + public void testAnalyze() throws Exception { + + RestHighLevelClient client = highLevelClient(); + + AnalyzeRequest noindexRequest = new AnalyzeRequest().text("One two three").analyzer("english"); + AnalyzeResponse noindexResponse = client.indices().analyze(noindexRequest, RequestOptions.DEFAULT); + + assertThat(noindexResponse.getTokens(), hasSize(3)); + + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index d9c018848d7e8..3a7ba72e05f87 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -26,6 +26,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -42,7 +44,7 @@ * A request to analyze a text associated with a specific index. Allow to provide * the actual analyzer name to perform the analysis with. */ -public class AnalyzeRequest extends SingleShardRequest { +public class AnalyzeRequest extends SingleShardRequest implements ToXContentObject { private String[] text; @@ -62,7 +64,7 @@ public class AnalyzeRequest extends SingleShardRequest { private String normalizer; - public static class NameOrDefinition implements Writeable { + public static class NameOrDefinition implements Writeable, ToXContentObject { // exactly one of these two members is not null public final String name; public final Settings definition; @@ -102,6 +104,19 @@ public void writeTo(StreamOutput out) throws IOException { Settings.writeSettingsToStream(definition, out); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (Strings.isNullOrEmpty(name) == false) { + return builder.value(name); + } + return definition.toXContent(builder, params); + } + + @Override + public boolean isFragment() { + return true; + } } public AnalyzeRequest() { @@ -260,4 +275,36 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(normalizer); } } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("text", text); + if (Strings.isNullOrEmpty(analyzer) == false) { + builder.field("analyzer", analyzer); + } + if (tokenizer != null) { + tokenizer.toXContent(builder, params); + } + if (tokenFilters.size() > 0) { + builder.field("filter", tokenFilters); + } + if (charFilters.size() > 0) { + builder.field("char_filter", charFilters); + } + if (Strings.isNullOrEmpty(field) == false) { + builder.field("field", field); + } + if (explain) { + builder.field("explain", true); + } + if (attributes.length > 0) { + builder.field("attributes", attributes); + } + if (Strings.isNullOrEmpty(normalizer) == false) { + builder.field("normalizer", normalizer); + } + return builder.endObject(); + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index 1e54def2385f8..b324a8ffbc3f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -25,12 +25,17 @@ import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; public class AnalyzeResponse extends ActionResponse implements Iterable, ToXContentObject { @@ -46,6 +51,25 @@ public static class AnalyzeToken implements Streamable, ToXContentObject { AnalyzeToken() { } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AnalyzeToken that = (AnalyzeToken) o; + return startOffset == that.startOffset && + endOffset == that.endOffset && + position == that.position && + positionLength == that.positionLength && + Objects.equals(term, that.term) && + Objects.equals(attributes, that.attributes) && + Objects.equals(type, that.type); + } + + @Override + public int hashCode() { + return Objects.hash(term, startOffset, endOffset, position, positionLength, attributes, type); + } + public AnalyzeToken(String term, int position, int startOffset, int endOffset, int positionLength, String type, Map attributes) { this.term = term; @@ -111,6 +135,46 @@ public static AnalyzeToken readAnalyzeToken(StreamInput in) throws IOException { return analyzeToken; } + public static AnalyzeToken readAnalyzeToken(XContentParser parser) throws IOException { + //ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + String field = null; + String term = ""; + int position = -1; + int startOffset = -1; + int endOffset = -1; + int positionLength = 1; + String type = ""; + Map attributes = new HashMap<>(); + for (XContentParser.Token t = parser.nextToken(); t != XContentParser.Token.END_OBJECT; t = parser.nextToken()) { + if (t == XContentParser.Token.FIELD_NAME) { + field = parser.currentName(); + continue; + } + if (Fields.TOKEN.equals(field)) { + term = parser.text(); + } + else if (Fields.POSITION.equals(field)) { + position = parser.intValue(); + } + else if (Fields.START_OFFSET.equals(field)) { + startOffset = parser.intValue(); + } + else if (Fields.END_OFFSET.equals(field)) { + endOffset = parser.intValue(); + } + else if (Fields.POSITION_LENGTH.equals(field)) { + positionLength = parser.intValue(); + } + else if (Fields.TYPE.equals(field)) { + type = parser.text(); + } + else { + attributes.put(field, parser.text()); + } + } + return new AnalyzeToken(term, position, startOffset, endOffset, positionLength, type, attributes); + } + @Override public void readFrom(StreamInput in) throws IOException { term = in.readString(); @@ -188,6 +252,29 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation); + String fieldName = parser.currentName(); + List tokens = null; + if (Fields.TOKENS.equals(fieldName)) { + tokens = readTokenList(parser); + } + return new AnalyzeResponse(tokens, null); + } + + private static List readTokenList(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + List tokens = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + tokens.add(AnalyzeToken.readAnalyzeToken(parser)); + } + return tokens; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); @@ -213,6 +300,20 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalStreamable(detail); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AnalyzeResponse that = (AnalyzeResponse) o; + return Objects.equals(detail, that.detail) && + Objects.equals(tokens, that.tokens); + } + + @Override + public int hashCode() { + return Objects.hash(detail, tokens); + } + static final class Fields { static final String TOKENS = "tokens"; static final String TOKEN = "token"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java new file mode 100644 index 0000000000000..169819e5c4ff7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -0,0 +1,54 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.analyze; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class AnalyzeResponseTests extends AbstractStreamableXContentTestCase { + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected AnalyzeResponse doParseInstance(XContentParser parser) throws IOException { + return AnalyzeResponse.fromXContent(parser); + } + + @Override + protected AnalyzeResponse createBlankInstance() { + return new AnalyzeResponse(); + } + + @Override + protected AnalyzeResponse createTestInstance() { + List tokens = new ArrayList<>(); + tokens.add(new AnalyzeResponse.AnalyzeToken("one", 0, 0, 3, 1, "", Collections.emptyMap())); + tokens.add(new AnalyzeResponse.AnalyzeToken("two", 1, 4, 7, 1, "", Collections.emptyMap())); + return new AnalyzeResponse(tokens, null); + } +} From 2563b4b58245bca14c1add225217e4a41a7d2479 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 21 Jun 2018 09:56:49 +0100 Subject: [PATCH 02/17] RequestConverters test --- .../elasticsearch/client/RequestConverters.java | 3 ++- .../client/RequestConvertersTests.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 826d2d7846d9e..f0dbe4bfa8df3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -894,11 +894,12 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) t } static Request analyze(AnalyzeRequest request) throws IOException { - EndpointBuilder builder = new EndpointBuilder().addPathPartAsIs("_analyze"); + EndpointBuilder builder = new EndpointBuilder(); String index = request.index(); if (index != null) { builder.addPathPart(index); } + builder.addPathPartAsIs("_analyze"); Request req = new Request(HttpGet.METHOD_NAME, builder.build()); req.setEntity(createEntity(request, REQUEST_BODY_CONTENT_TYPE)); return req; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 60f427b490462..46eac4a0e3f8e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -1948,6 +1949,22 @@ public void testGetTemplateRequest() throws Exception { assertThat(request.getEntity(), nullValue()); } + public void testAnalyzeRequest() throws Exception { + AnalyzeRequest indexAnalyzeRequest = new AnalyzeRequest() + .text("Here is some text") + .index("test_index") + .analyzer("test_analyzer"); + + Request request = RequestConverters.analyze(indexAnalyzeRequest); + assertThat(request.getEndpoint(), equalTo("/test_index/_analyze")); + assertThat(request.getEntity(), notNullValue()); + + AnalyzeRequest analyzeRequest = new AnalyzeRequest() + .text("more text") + .analyzer("test_analyzer"); + assertThat(RequestConverters.analyze(analyzeRequest).getEndpoint(), equalTo("/_analyze")); + } + private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException { BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, REQUEST_BODY_CONTENT_TYPE, false); assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue()); From 3bd541c35b6d6e35c8e081100e0070edc5d11b24 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 25 Jun 2018 11:25:38 +0100 Subject: [PATCH 03/17] Add detailed response --- .../indices/analyze/AnalyzeResponse.java | 24 ++- .../analyze/DetailAnalyzeResponse.java | 148 ++++++++++++++++++ .../indices/analyze/AnalyzeResponseTests.java | 22 ++- 3 files changed, 184 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index b324a8ffbc3f4..ee797e3ba4eab 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -136,7 +137,7 @@ public static AnalyzeToken readAnalyzeToken(StreamInput in) throws IOException { } public static AnalyzeToken readAnalyzeToken(XContentParser parser) throws IOException { - //ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); String field = null; String term = ""; int position = -1; @@ -257,13 +258,24 @@ public static AnalyzeResponse fromXContent(XContentParser parser) throws IOExcep parser.nextToken(); } ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); - ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation); - String fieldName = parser.currentName(); + String fieldName = null; List tokens = null; - if (Fields.TOKENS.equals(fieldName)) { - tokens = readTokenList(parser); + DetailAnalyzeResponse dar = null; + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + fieldName = parser.currentName(); + if (Fields.TOKENS.equals(fieldName)) { + tokens = readTokenList(parser); + } + else if (Fields.DETAIL.equals(fieldName)) { + dar = DetailAnalyzeResponse.fromXContent(parser); + } + else { + throw new ParsingException(parser.getTokenLocation(), "Unexpected field name [" + fieldName + "] in AnalyzeResponse"); + } + } } - return new AnalyzeResponse(tokens, null); + return new AnalyzeResponse(tokens, dar); } private static List readTokenList(XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index c080a01a98168..0c306ed2ce894 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.analyze; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -27,14 +28,40 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; public class DetailAnalyzeResponse implements Streamable, ToXContentFragment { DetailAnalyzeResponse() { } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DetailAnalyzeResponse that = (DetailAnalyzeResponse) o; + return customAnalyzer == that.customAnalyzer && + Objects.equals(analyzer, that.analyzer) && + Arrays.equals(charfilters, that.charfilters) && + Objects.equals(tokenizer, that.tokenizer) && + Arrays.equals(tokenfilters, that.tokenfilters); + } + + @Override + public int hashCode() { + int result = Objects.hash(customAnalyzer, analyzer, tokenizer); + result = 31 * result + Arrays.hashCode(charfilters); + result = 31 * result + Arrays.hashCode(tokenfilters); + return result; + } + private boolean customAnalyzer = false; private AnalyzeTokenList analyzer; private CharFilteredText[] charfilters; @@ -66,6 +93,7 @@ public AnalyzeTokenList analyzer() { } public DetailAnalyzeResponse analyzer(AnalyzeTokenList analyzer) { + this.customAnalyzer = false; this.analyzer = analyzer; return this; } @@ -75,6 +103,7 @@ public CharFilteredText[] charfilters() { } public DetailAnalyzeResponse charfilters(CharFilteredText[] charfilters) { + this.customAnalyzer = true; this.charfilters = charfilters; return this; } @@ -84,6 +113,7 @@ public AnalyzeTokenList tokenizer() { } public DetailAnalyzeResponse tokenizer(AnalyzeTokenList tokenizer) { + this.customAnalyzer = true; this.tokenizer = tokenizer; return this; } @@ -93,6 +123,7 @@ public AnalyzeTokenList[] tokenfilters() { } public DetailAnalyzeResponse tokenfilters(AnalyzeTokenList[] tokenfilters) { + this.customAnalyzer = true; this.tokenfilters = tokenfilters; return this; } @@ -131,6 +162,46 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static DetailAnalyzeResponse fromXContent(XContentParser parser) throws IOException { + String currentName = null; + boolean customAnalyzer = false; + AnalyzeTokenList analyzer = null; + CharFilteredText[] charFilteredText = null; + AnalyzeTokenList tokenizer = null; + AnalyzeTokenList[] tokenfilters = null; + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + currentName = parser.currentName(); + } + else if (Fields.CUSTOM_ANALYZER.equals(currentName)) { + customAnalyzer = parser.booleanValue(); + } + else if (Fields.ANALYZER.equals(currentName)) { + analyzer = AnalyzeTokenList.fromXContent(parser); + } + else if (Fields.CHARFILTERS.equals(currentName)) { + List charfilters = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + charfilters.add(CharFilteredText.fromXContent(parser)); + } + charFilteredText = charfilters.toArray(new CharFilteredText[0]); + } + else if (Fields.TOKENIZER.equals(currentName)) { + tokenizer = AnalyzeTokenList.fromXContent(parser); + } + else if (Fields.TOKENFILTERS.equals(currentName)) { + List filters = new ArrayList<>(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + filters.add(AnalyzeTokenList.fromXContent(parser)); + } + tokenfilters = filters.toArray(new AnalyzeTokenList[0]); + } + } + return new DetailAnalyzeResponse(customAnalyzer, analyzer, charFilteredText, tokenizer, tokenfilters); + } + static final class Fields { static final String NAME = "name"; static final String FILTERED_TEXT = "filtered_text"; @@ -195,6 +266,22 @@ public static class AnalyzeTokenList implements Streamable, ToXContentObject { private String name; private AnalyzeResponse.AnalyzeToken[] tokens; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AnalyzeTokenList that = (AnalyzeTokenList) o; + return Objects.equals(name, that.name) && + Arrays.equals(tokens, that.tokens); + } + + @Override + public int hashCode() { + int result = Objects.hash(name); + result = 31 * result + Arrays.hashCode(tokens); + return result; + } + AnalyzeTokenList() { } @@ -235,6 +322,29 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static AnalyzeTokenList fromXContent(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + String name = null; + List tokens = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + // parser.nextToken(); + String field = parser.currentName(); + if (Fields.NAME.equals(field)) { + parser.nextToken(); + name = parser.text(); + } + else if (AnalyzeResponse.Fields.TOKENS.equals(field)) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + tokens.add(AnalyzeResponse.AnalyzeToken.readAnalyzeToken(parser)); + } + } + } + } + return new AnalyzeTokenList(name, tokens.toArray(new AnalyzeResponse.AnalyzeToken[0])); + } + @Override public void readFrom(StreamInput in) throws IOException { name = in.readString(); @@ -293,6 +403,28 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static CharFilteredText fromXContent(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + String name = null; + List texts = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + if (Fields.NAME.equals(parser.currentName())) { + parser.nextToken(); + name = parser.text(); + } + else if (Fields.FILTERED_TEXT.equals(parser.currentName())) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + texts.add(parser.text()); + } + } + } + } + //XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation); + return new CharFilteredText(name, texts.toArray(new String[0])); + } + public static CharFilteredText readCharFilteredText(StreamInput in) throws IOException { CharFilteredText text = new CharFilteredText(); text.readFrom(in); @@ -310,5 +442,21 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(name); out.writeStringArray(texts); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CharFilteredText that = (CharFilteredText) o; + return Objects.equals(name, that.name) && + Arrays.equals(texts, that.texts); + } + + @Override + public int hashCode() { + int result = Objects.hash(name); + result = 31 * result + Arrays.hashCode(texts); + return result; + } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 169819e5c4ff7..50126466041a7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -19,11 +19,15 @@ package org.elasticsearch.action.admin.indices.analyze; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -46,9 +50,19 @@ protected AnalyzeResponse createBlankInstance() { @Override protected AnalyzeResponse createTestInstance() { - List tokens = new ArrayList<>(); - tokens.add(new AnalyzeResponse.AnalyzeToken("one", 0, 0, 3, 1, "", Collections.emptyMap())); - tokens.add(new AnalyzeResponse.AnalyzeToken("two", 1, 4, 7, 1, "", Collections.emptyMap())); - return new AnalyzeResponse(tokens, null); + AnalyzeResponse.AnalyzeToken[] tokens = new AnalyzeResponse.AnalyzeToken[]{ + new AnalyzeResponse.AnalyzeToken("one", 0, 0, 3, 1, "", Collections.emptyMap()), + new AnalyzeResponse.AnalyzeToken("two", 1, 4, 7, 1, "", Collections.emptyMap()) + }; + DetailAnalyzeResponse dar = new DetailAnalyzeResponse(); + dar.charfilters(new DetailAnalyzeResponse.CharFilteredText[]{ + new DetailAnalyzeResponse.CharFilteredText("my_charfilter", new String[]{"one two"}) + }); + dar.tokenizer(new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenizer", tokens)); + dar.tokenfilters(new DetailAnalyzeResponse.AnalyzeTokenList[]{ + new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_1", tokens), + new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_2", tokens) + }); + return new AnalyzeResponse(Arrays.asList(tokens), dar); } } From 722214edf25f39f1af1e3a6269f3e4d70b1afedb Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 26 Jun 2018 13:18:31 +0100 Subject: [PATCH 04/17] docs --- .../elasticsearch/client/IndicesClient.java | 17 +++ .../elasticsearch/client/IndicesClientIT.java | 7 +- .../IndicesClientDocumentationIT.java | 126 ++++++++++++++++++ .../high-level/indices/analyze.asciidoc | 112 ++++++++++++++++ .../high-level/supported-apis.asciidoc | 1 + 5 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 docs/java-rest/high-level/indices/analyze.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 4f0df184cc82f..2f4bf8e4a8220 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -724,11 +724,28 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); } + /** + * Calls the analyze API + * + * See Analyze API on elastic.co + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + */ public AnalyzeResponse analyze(AnalyzeRequest request, RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::analyze, options, AnalyzeResponse::fromXContent, emptySet()); } + /** + * Asynchronously calls the analyze API + * + * See Analyze API on elastic.co + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ public void analyzeAsync(AnalyzeRequest request, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::analyze, options, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 2778c6ea9a168..aed16cbbb70fe 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -1248,9 +1248,14 @@ public void testAnalyze() throws Exception { RestHighLevelClient client = highLevelClient(); AnalyzeRequest noindexRequest = new AnalyzeRequest().text("One two three").analyzer("english"); - AnalyzeResponse noindexResponse = client.indices().analyze(noindexRequest, RequestOptions.DEFAULT); + AnalyzeResponse noindexResponse = execute(noindexRequest, client.indices()::analyze, client.indices()::analyzeAsync); assertThat(noindexResponse.getTokens(), hasSize(3)); + AnalyzeRequest detailsRequest = new AnalyzeRequest().text("One two three").analyzer("english").explain(true); + AnalyzeResponse detailsResponse = execute(detailsRequest, client.indices()::analyze, client.indices()::analyzeAsync); + + assertNotNull(detailsResponse.detail()); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 9cc28152d03e3..c7651fa7d7b7d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -27,6 +27,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; +import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; +import org.elasticsearch.action.admin.indices.analyze.DetailAnalyzeResponse; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; @@ -2211,4 +2214,127 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + + public void testAnalyze() throws IOException, InterruptedException { + + RestHighLevelClient client = highLevelClient(); + + { + // tag::analyze-builtin-request + AnalyzeRequest request = new AnalyzeRequest(); + request.text("Some text to analyze", "Some more text to analyze"); // <1> + request.analyzer("english"); // <2> + // end::analyze-builtin-request + } + + { + // tag::analyze-custom-request + AnalyzeRequest request = new AnalyzeRequest(); + request.text("Some text to analyze"); + request.addCharFilter("html_strip"); // <1> + request.tokenizer("standard"); // <2> + request.addTokenFilter("lowercase"); // <3> + + Map stopFilter = new HashMap<>(); + stopFilter.put("type", "stop"); + stopFilter.put("stopwords", new String[]{ "to" }); // <4> + request.addTokenFilter(stopFilter); // <5> + // end::analyze-custom-request + } + + { + // tag::analyze-custom-normalizer-request + AnalyzeRequest request = new AnalyzeRequest(); + request.text("BaR"); + request.addCharFilter("html_strip"); + request.addTokenFilter("lowercase"); + // end::analyze-custom-normalizer-request + + // tag::analyze-request-explain + request.explain(true); + request.attributes("keyword", "type"); + // end::analyze-request-explain + + // tag::analyze-request-sync + AnalyzeResponse response = client.indices().analyze(request, RequestOptions.DEFAULT); + // end::analyze-request-sync + + // tag::analyze-response + List tokens = response.getTokens(); // <1> + DetailAnalyzeResponse detail = response.detail(); // <2> + // end::analyze-response + + assertEquals(tokens.size(), 1); + assertEquals(tokens.get(0).getTerm(), "bar"); + assertNotNull(detail.tokenizer()); + } + + CreateIndexRequest req = new CreateIndexRequest("my_index"); + CreateIndexResponse resp = client.indices().create(req, RequestOptions.DEFAULT); + assertTrue(resp.isAcknowledged()); + + PutMappingRequest pmReq = new PutMappingRequest() + .indices("my_index") + .source("my_field", "type=text,analyzer=english"); + PutMappingResponse pmResp = client.indices().putMapping(pmReq, RequestOptions.DEFAULT); + assertTrue(pmResp.isAcknowledged()); + + { + // tag::analyze-index-request + AnalyzeRequest request = new AnalyzeRequest(); + request.index("my_index"); // <1> + request.analyzer("my_analyzer"); // <2> + request.text("some text to analyze"); + // end::analyze-index-request + + // tag::analyze-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(AnalyzeResponse analyzeTokens) { + + } + + @Override + public void onFailure(Exception e) { + + } + }; + // end::analyze-execute-listener + + // Use a blocking listener in the test + final CountDownLatch latch = new CountDownLatch(1); + final ActionListener blockingListener = new LatchedActionListener<>(listener, latch); + listener = ActionListener.wrap(r -> { + assertThat(r.getTokens(), hasSize(4)); + }, e-> { + blockingListener.onFailure(e); + fail("should not fail"); + }); + + // tag::analyze-request-async + client.indices().analyzeAsync(request, RequestOptions.DEFAULT, listener); + // end::analyze-request-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + { + // tag::analyze-index-normalizer-request + AnalyzeRequest request = new AnalyzeRequest(); + request.index("my_index"); // <1> + request.normalizer("my_normalizer"); // <2> + request.text("some text to analyze"); + // end::analyze-index-normalizer-request + } + + { + // tag::analyze-field-request + AnalyzeRequest request = new AnalyzeRequest(); + request.index("my_index"); + request.field("my_field"); + request.text("some text to analyze"); + // end::analyze-field-request + } + + } } diff --git a/docs/java-rest/high-level/indices/analyze.asciidoc b/docs/java-rest/high-level/indices/analyze.asciidoc new file mode 100644 index 0000000000000..ba18ef13e4451 --- /dev/null +++ b/docs/java-rest/high-level/indices/analyze.asciidoc @@ -0,0 +1,112 @@ +[[java-rest-high-analyze]] +=== Analyze API + +[[java-rest-high-analyze-request]] +==== Analyze Request + +An `AnalyzeRequest` contains the text to analyze, and one of several options to +specify how the analysis should be performed. + +The simplest version uses a built-in analyzer: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-builtin-request] +--------------------------------------------------- +<1> The text to include. Multiple strings are treated as a multi-valued field +<2> A built-in analyzer + +You can configure a custom analyzer: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-custom-request] +--------------------------------------------------- +<1> Configure char filters +<2> Configure the tokenizer +<3> Add a built-in tokenfilter +<4> Configuration for a custom tokenfilter +<5> Add the custom tokenfilter + +You can also build a custom normalizer, by including only charfilters and +tokenfilters: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-custom-normalizer-request] +--------------------------------------------------- + +You can analyze text using an analyzer defined in an existing index: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-index-request] +--------------------------------------------------- +<1> The index containing the mappings +<2> The analyzer defined on this index to use + +Or you can use a normalizer: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-index-normalizer-request] +--------------------------------------------------- +<1> The index containing the mappings +<2> The normalizer defined on this index to use + +You can analyze text using the mappings for a particular field in an index: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-field-request] +--------------------------------------------------- + +==== Optional arguemnts +The following arguments can also optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-request-explain] +--------------------------------------------------- +<1> Setting `explain` to true will add further details to the response +<2> Setting `attributes` allows you to return only token attributes that you are +interested in + +[[java-rest-high-analyze-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-request-sync] +--------------------------------------------------- + +[[java-rest-high-analyze-async]] +==== Asynchronous Execution + +The asynchronous execution of an analyze request requires both the `AnalyzeRequest` +instance and an `ActionListener` instance to be passed to the asyncronous method: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-request-async] +--------------------------------------------------- + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method if the +execution successfully completed or using the `onFailure` method if it failed. + +A typical listener for `AnalyzeResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-execute-listener] +--------------------------------------------------- + +[[java-rest-high-analyze-response]] +==== Analyze Response + +The returned `AnalyzeResponse` allows you to retrieve details of the analysis as +follows: +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-response] +--------------------------------------------------- +<1> `AnalyzeToken` holds information about the individual tokens produced by analysis +<2> `DetailAnalyzeResponse` holds more detailed information about tokens produced by +the various substeps in the analysis chain. If `explain` was set to `false` in the +request, this method will return `null` \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 17acc8f13c04d..8d77ffd4fbe4e 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -83,6 +83,7 @@ Alias Management:: * <> * <> +include::indices/analyze.asciidoc[] include::indices/create_index.asciidoc[] include::indices/delete_index.asciidoc[] include::indices/indices_exists.asciidoc[] From 2c4ee452f0682628f41f091e9b246bce64d55e52 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 26 Jun 2018 13:32:08 +0100 Subject: [PATCH 05/17] checkstyle --- .../action/admin/indices/analyze/AnalyzeRequest.java | 2 +- .../admin/indices/analyze/DetailAnalyzeResponse.java | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 3a7ba72e05f87..8b7b8b6719eff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -186,6 +185,7 @@ public AnalyzeRequest addCharFilter(String charFilter) { this.charFilters.add(new NameOrDefinition(charFilter)); return this; } + public List charFilters() { return this.charFilters; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index 0c306ed2ce894..e6d50985fe682 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.analyze; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -335,7 +334,8 @@ public static AnalyzeTokenList fromXContent(XContentParser parser) throws IOExce name = parser.text(); } else if (AnalyzeResponse.Fields.TOKENS.equals(field)) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), + parser::getTokenLocation); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { tokens.add(AnalyzeResponse.AnalyzeToken.readAnalyzeToken(parser)); } @@ -404,7 +404,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static CharFilteredText fromXContent(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), + parser::getTokenLocation); String name = null; List texts = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -414,7 +415,8 @@ public static CharFilteredText fromXContent(XContentParser parser) throws IOExce name = parser.text(); } else if (Fields.FILTERED_TEXT.equals(parser.currentName())) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), + parser::getTokenLocation); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { texts.add(parser.text()); } From c8c5008b2114e2041f56a2d9168222200c9bf0c9 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 26 Jun 2018 15:01:40 +0100 Subject: [PATCH 06/17] checkstyle --- .../action/admin/indices/analyze/AnalyzeResponseTests.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 50126466041a7..b2f9cef98d163 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -19,17 +19,12 @@ package org.elasticsearch.action.admin.indices.analyze; -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; public class AnalyzeResponseTests extends AbstractStreamableXContentTestCase { From 69e1119d99e7b2999280ae5e07de4bc44fcbb814 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 27 Jun 2018 15:24:51 +0100 Subject: [PATCH 07/17] docs fix --- .../client/documentation/IndicesClientDocumentationIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index c7651fa7d7b7d..f68c8923b9f70 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2246,7 +2246,6 @@ public void testAnalyze() throws IOException, InterruptedException { // tag::analyze-custom-normalizer-request AnalyzeRequest request = new AnalyzeRequest(); request.text("BaR"); - request.addCharFilter("html_strip"); request.addTokenFilter("lowercase"); // end::analyze-custom-normalizer-request From 52d2c41a4df59d0d38ba128c720a66c5a879ad0a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 27 Jun 2018 17:17:25 +0100 Subject: [PATCH 08/17] Use ConstructingObjectParser --- .../client/RequestConvertersTests.java | 2 +- .../IndicesClientDocumentationIT.java | 2 +- .../admin/indices/analyze/AnalyzeRequest.java | 9 +- .../indices/analyze/AnalyzeResponse.java | 43 ++----- .../analyze/DetailAnalyzeResponse.java | 116 ++++++------------ 5 files changed, 54 insertions(+), 118 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 58d8f9eca422c..93ae276d2a621 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1959,7 +1959,7 @@ public void testAnalyzeRequest() throws Exception { Request request = RequestConverters.analyze(indexAnalyzeRequest); assertThat(request.getEndpoint(), equalTo("/test_index/_analyze")); - assertThat(request.getEntity(), notNullValue()); + assertToXContentBody(indexAnalyzeRequest, request.getEntity()); AnalyzeRequest analyzeRequest = new AnalyzeRequest() .text("more text") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index f68c8923b9f70..0060ef9cf7ca6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2320,7 +2320,7 @@ public void onFailure(Exception e) { { // tag::analyze-index-normalizer-request AnalyzeRequest request = new AnalyzeRequest(); - request.index("my_index"); // <1> + request.index("my_index"); // <1> request.normalizer("my_normalizer"); // <2> request.text("some text to analyze"); // end::analyze-index-normalizer-request diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java index 8b7b8b6719eff..09686025e9da9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -63,7 +64,7 @@ public class AnalyzeRequest extends SingleShardRequest implement private String normalizer; - public static class NameOrDefinition implements Writeable, ToXContentObject { + public static class NameOrDefinition implements Writeable, ToXContentFragment { // exactly one of these two members is not null public final String name; public final Settings definition; @@ -106,16 +107,12 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (Strings.isNullOrEmpty(name) == false) { + if (definition == null) { return builder.value(name); } return definition.toXContent(builder, params); } - @Override - public boolean isFragment() { - return true; - } } public AnalyzeRequest() { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index ee797e3ba4eab..ed07c5ad5ae1a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -20,10 +20,13 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -36,6 +39,7 @@ import java.util.Map; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; public class AnalyzeResponse extends ActionResponse implements Iterable, ToXContentObject { @@ -136,7 +140,7 @@ public static AnalyzeToken readAnalyzeToken(StreamInput in) throws IOException { return analyzeToken; } - public static AnalyzeToken readAnalyzeToken(XContentParser parser) throws IOException { + public static AnalyzeToken fromXContent(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); String field = null; String term = ""; @@ -253,38 +257,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException { - if (parser.currentToken() == null) { - parser.nextToken(); - } - ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); - String fieldName = null; - List tokens = null; - DetailAnalyzeResponse dar = null; - for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { - if (token == XContentParser.Token.FIELD_NAME) { - fieldName = parser.currentName(); - if (Fields.TOKENS.equals(fieldName)) { - tokens = readTokenList(parser); - } - else if (Fields.DETAIL.equals(fieldName)) { - dar = DetailAnalyzeResponse.fromXContent(parser); - } - else { - throw new ParsingException(parser.getTokenLocation(), "Unexpected field name [" + fieldName + "] in AnalyzeResponse"); - } - } - } - return new AnalyzeResponse(tokens, dar); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("analyze_response", + args -> new AnalyzeResponse((List) args[0], (DetailAnalyzeResponse) args[1])); + static { + PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS)); + PARSER.declareObject(constructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL)); } - private static List readTokenList(XContentParser parser) throws IOException { - ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); - List tokens = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - tokens.add(AnalyzeToken.readAnalyzeToken(parser)); - } - return tokens; + public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index e6d50985fe682..f77d56938e650 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -20,10 +20,12 @@ package org.elasticsearch.action.admin.indices.analyze; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,6 +38,9 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + public class DetailAnalyzeResponse implements Streamable, ToXContentFragment { DetailAnalyzeResponse() { @@ -161,44 +166,21 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("detail", + args -> new DetailAnalyzeResponse((boolean)args[0], (AnalyzeTokenList)args[1], + ((List)args[2]).toArray(new CharFilteredText[0]), + (AnalyzeTokenList)args[3], + ((List)args[4]).toArray(new AnalyzeTokenList[0]))); + static { + PARSER.declareBoolean(constructorArg(), new ParseField(Fields.CUSTOM_ANALYZER)); + PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.ANALYZER)); + PARSER.declareObjectArray(optionalConstructorArg(), CharFilteredText.PARSER, new ParseField(Fields.CHARFILTERS)); + PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.TOKENIZER)); + PARSER.declareObjectArray(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.TOKENFILTERS)); + } + public static DetailAnalyzeResponse fromXContent(XContentParser parser) throws IOException { - String currentName = null; - boolean customAnalyzer = false; - AnalyzeTokenList analyzer = null; - CharFilteredText[] charFilteredText = null; - AnalyzeTokenList tokenizer = null; - AnalyzeTokenList[] tokenfilters = null; - for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { - if (token == XContentParser.Token.FIELD_NAME) { - currentName = parser.currentName(); - } - else if (Fields.CUSTOM_ANALYZER.equals(currentName)) { - customAnalyzer = parser.booleanValue(); - } - else if (Fields.ANALYZER.equals(currentName)) { - analyzer = AnalyzeTokenList.fromXContent(parser); - } - else if (Fields.CHARFILTERS.equals(currentName)) { - List charfilters = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - charfilters.add(CharFilteredText.fromXContent(parser)); - } - charFilteredText = charfilters.toArray(new CharFilteredText[0]); - } - else if (Fields.TOKENIZER.equals(currentName)) { - tokenizer = AnalyzeTokenList.fromXContent(parser); - } - else if (Fields.TOKENFILTERS.equals(currentName)) { - List filters = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - filters.add(AnalyzeTokenList.fromXContent(parser)); - } - tokenfilters = filters.toArray(new AnalyzeTokenList[0]); - } - } - return new DetailAnalyzeResponse(customAnalyzer, analyzer, charFilteredText, tokenizer, tokenfilters); + return PARSER.parse(parser, null); } static final class Fields { @@ -321,28 +303,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("token_list", + args -> new AnalyzeTokenList((String)args[0], + ((List)args[1]).toArray(new AnalyzeResponse.AnalyzeToken[0]))); + static { + PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); + PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeResponse.AnalyzeToken.fromXContent(p), + new ParseField(AnalyzeResponse.Fields.TOKENS)); + } + public static AnalyzeTokenList fromXContent(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); - String name = null; - List tokens = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - // parser.nextToken(); - String field = parser.currentName(); - if (Fields.NAME.equals(field)) { - parser.nextToken(); - name = parser.text(); - } - else if (AnalyzeResponse.Fields.TOKENS.equals(field)) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), - parser::getTokenLocation); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - tokens.add(AnalyzeResponse.AnalyzeToken.readAnalyzeToken(parser)); - } - } - } - } - return new AnalyzeTokenList(name, tokens.toArray(new AnalyzeResponse.AnalyzeToken[0])); + return PARSER.parse(parser, null); } @Override @@ -403,28 +374,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("char_filtered_text", + args -> new CharFilteredText((String)args[0], ((List)args[1]).toArray(new String[0]))); + static { + PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); + PARSER.declareStringArray(constructorArg(), new ParseField(Fields.FILTERED_TEXT)); + } + public static CharFilteredText fromXContent(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), - parser::getTokenLocation); - String name = null; - List texts = new ArrayList<>(); - while (parser.nextToken() != XContentParser.Token.END_OBJECT) { - if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { - if (Fields.NAME.equals(parser.currentName())) { - parser.nextToken(); - name = parser.text(); - } - else if (Fields.FILTERED_TEXT.equals(parser.currentName())) { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), - parser::getTokenLocation); - while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - texts.add(parser.text()); - } - } - } - } - //XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation); - return new CharFilteredText(name, texts.toArray(new String[0])); + return PARSER.parse(parser, null); } public static CharFilteredText readCharFilteredText(StreamInput in) throws IOException { From 74352288b8add8b1c01fdef3eb07a14f3145b1ef Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 27 Jun 2018 18:00:13 +0100 Subject: [PATCH 09/17] handle unknown fields --- .../admin/indices/analyze/AnalyzeResponse.java | 13 +++++++++++-- .../indices/analyze/DetailAnalyzeResponse.java | 6 +++--- .../admin/indices/analyze/AnalyzeResponseTests.java | 7 +++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index ed07c5ad5ae1a..e772760e1903b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -174,7 +174,16 @@ else if (Fields.TYPE.equals(field)) { type = parser.text(); } else { - attributes.put(field, parser.text()); + if (t == XContentParser.Token.VALUE_STRING) { + attributes.put(field, parser.text()); + } + else if (t == XContentParser.Token.VALUE_NUMBER) { + attributes.put(field, parser.numberValue()); + } + else if (t == XContentParser.Token.VALUE_BOOLEAN) { + attributes.put(field, parser.booleanValue()); + } + // TODO: Can we parse arbitrary objects into maps? } } return new AnalyzeToken(term, position, startOffset, endOffset, positionLength, type, attributes); @@ -258,7 +267,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("analyze_response", - args -> new AnalyzeResponse((List) args[0], (DetailAnalyzeResponse) args[1])); + true, args -> new AnalyzeResponse((List) args[0], (DetailAnalyzeResponse) args[1])); static { PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS)); PARSER.declareObject(constructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL)); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index f77d56938e650..c4de7155e34c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -167,7 +167,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("detail", - args -> new DetailAnalyzeResponse((boolean)args[0], (AnalyzeTokenList)args[1], + true, args -> new DetailAnalyzeResponse((boolean)args[0], (AnalyzeTokenList)args[1], ((List)args[2]).toArray(new CharFilteredText[0]), (AnalyzeTokenList)args[3], ((List)args[4]).toArray(new AnalyzeTokenList[0]))); @@ -304,7 +304,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("token_list", - args -> new AnalyzeTokenList((String)args[0], + true, args -> new AnalyzeTokenList((String)args[0], ((List)args[1]).toArray(new AnalyzeResponse.AnalyzeToken[0]))); static { PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); @@ -375,7 +375,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("char_filtered_text", - args -> new CharFilteredText((String)args[0], ((List)args[1]).toArray(new String[0]))); + true, args -> new CharFilteredText((String)args[0], ((List)args[1]).toArray(new String[0]))); static { PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); PARSER.declareStringArray(constructorArg(), new ParseField(Fields.FILTERED_TEXT)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index b2f9cef98d163..a9ebd9808e1d9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -25,12 +25,15 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Predicate; public class AnalyzeResponseTests extends AbstractStreamableXContentTestCase { @Override - protected boolean supportsUnknownFields() { - return false; + protected Predicate getRandomFieldsExcludeFilter() { + return s -> s.contains("tokens."); } @Override From ceb6601d593dc10a168598227448dd5007721609 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 28 Jun 2018 10:34:06 +0100 Subject: [PATCH 10/17] Add randomization to building responses for tests --- .../indices/analyze/AnalyzeResponse.java | 23 ++++++-- .../analyze/DetailAnalyzeResponse.java | 29 ++++++--- .../indices/analyze/AnalyzeResponseTests.java | 59 ++++++++++++++----- 3 files changed, 80 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index e772760e1903b..0fe42a472cd0a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -21,12 +21,11 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,11 +34,14 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; public class AnalyzeResponse extends ActionResponse implements Iterable, ToXContentObject { @@ -126,7 +128,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.POSITION_LENGTH, positionLength); } if (attributes != null && !attributes.isEmpty()) { - for (Map.Entry entity : attributes.entrySet()) { + Map sortedAttributes = new TreeMap<>(attributes); + for (Map.Entry entity : sortedAttributes.entrySet()) { builder.field(entity.getKey(), entity.getValue()); } } @@ -203,8 +206,11 @@ public void readFrom(StreamInput in) throws IOException { positionLength = 1; } } + else { + positionLength = 1; + } type = in.readOptionalString(); - attributes = (Map) in.readGenericValue(); + attributes = in.readMap(); } @Override @@ -217,7 +223,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(positionLength > 1 ? positionLength : null); } out.writeOptionalString(type); - out.writeGenericValue(attributes); + out.writeMapWithConsistentOrder(attributes); } } @@ -270,7 +276,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws true, args -> new AnalyzeResponse((List) args[0], (DetailAnalyzeResponse) args[1])); static { PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS)); - PARSER.declareObject(constructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL)); + PARSER.declareObject(optionalConstructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL)); } public static AnalyzeResponse fromXContent(XContentParser parser) throws IOException { @@ -316,6 +322,11 @@ public int hashCode() { return Objects.hash(detail, tokens); } + @Override + public String toString() { + return Strings.toString(this, true, true); + } + static final class Fields { static final String TOKENS = "tokens"; static final String TOKEN = "token"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index c4de7155e34c9..b9c305596ae6f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -30,10 +30,9 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; -import java.util.ArrayList; +import java.lang.reflect.Array; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -166,11 +165,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @SuppressWarnings("unchecked") + private static T[] fromList(Class clazz, List list) { + if (list == null) { + return null; + } + return list.toArray((T[])Array.newInstance(clazz, 0)); + } + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("detail", - true, args -> new DetailAnalyzeResponse((boolean)args[0], (AnalyzeTokenList)args[1], - ((List)args[2]).toArray(new CharFilteredText[0]), - (AnalyzeTokenList)args[3], - ((List)args[4]).toArray(new AnalyzeTokenList[0]))); + true, args -> new DetailAnalyzeResponse((boolean) args[0], (AnalyzeTokenList) args[1], + fromList(CharFilteredText.class, (List)args[2]), + (AnalyzeTokenList) args[3], + fromList(AnalyzeTokenList.class, (List)args[4]))); + static { PARSER.declareBoolean(constructorArg(), new ParseField(Fields.CUSTOM_ANALYZER)); PARSER.declareObject(optionalConstructorArg(), AnalyzeTokenList.PARSER, new ParseField(Fields.ANALYZER)); @@ -304,8 +312,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("token_list", - true, args -> new AnalyzeTokenList((String)args[0], - ((List)args[1]).toArray(new AnalyzeResponse.AnalyzeToken[0]))); + true, args -> new AnalyzeTokenList((String) args[0], + fromList(AnalyzeResponse.AnalyzeToken.class, (List)args[1]))); + static { PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeResponse.AnalyzeToken.fromXContent(p), @@ -345,6 +354,7 @@ public void writeTo(StreamOutput out) throws IOException { public static class CharFilteredText implements Streamable, ToXContentObject { private String name; private String[] texts; + CharFilteredText() { } @@ -375,7 +385,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("char_filtered_text", - true, args -> new CharFilteredText((String)args[0], ((List)args[1]).toArray(new String[0]))); + true, args -> new CharFilteredText((String) args[0], ((List) args[1]).toArray(new String[0]))); + static { PARSER.declareString(constructorArg(), new ParseField(Fields.NAME)); PARSER.declareStringArray(constructorArg(), new ParseField(Fields.FILTERED_TEXT)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index a9ebd9808e1d9..dab88ae3fc124 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -24,9 +24,9 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.function.Predicate; public class AnalyzeResponseTests extends AbstractStreamableXContentTestCase { @@ -48,19 +48,46 @@ protected AnalyzeResponse createBlankInstance() { @Override protected AnalyzeResponse createTestInstance() { - AnalyzeResponse.AnalyzeToken[] tokens = new AnalyzeResponse.AnalyzeToken[]{ - new AnalyzeResponse.AnalyzeToken("one", 0, 0, 3, 1, "", Collections.emptyMap()), - new AnalyzeResponse.AnalyzeToken("two", 1, 4, 7, 1, "", Collections.emptyMap()) - }; - DetailAnalyzeResponse dar = new DetailAnalyzeResponse(); - dar.charfilters(new DetailAnalyzeResponse.CharFilteredText[]{ - new DetailAnalyzeResponse.CharFilteredText("my_charfilter", new String[]{"one two"}) - }); - dar.tokenizer(new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenizer", tokens)); - dar.tokenfilters(new DetailAnalyzeResponse.AnalyzeTokenList[]{ - new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_1", tokens), - new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_2", tokens) - }); + int tokenCount = randomIntBetween(1, 30); + AnalyzeResponse.AnalyzeToken[] tokens = new AnalyzeResponse.AnalyzeToken[tokenCount]; + for (int i = 0; i < tokenCount; i++) { + tokens[i] = randomToken(); + } + DetailAnalyzeResponse dar = null; + if (randomBoolean()) { + dar = new DetailAnalyzeResponse(); + if (randomBoolean()) { + dar.charfilters(new DetailAnalyzeResponse.CharFilteredText[]{ + new DetailAnalyzeResponse.CharFilteredText("my_charfilter", new String[]{"one two"}) + }); + } + dar.tokenizer(new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenizer", tokens)); + if (randomBoolean()) { + dar.tokenfilters(new DetailAnalyzeResponse.AnalyzeTokenList[]{ + new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_1", tokens), + new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_2", tokens) + }); + } + } return new AnalyzeResponse(Arrays.asList(tokens), dar); } + + private AnalyzeResponse.AnalyzeToken randomToken() { + String token = randomAlphaOfLengthBetween(1, 20); + int position = randomIntBetween(0, 1000); + int startOffset = randomIntBetween(0, 1000); + int endOffset = randomIntBetween(0, 1000); + int posLength = randomIntBetween(1, 5); + String type = randomAlphaOfLengthBetween(1, 20); + Map extras = new HashMap<>(); + if (randomBoolean()) { + int entryCount = randomInt(6); + for (int i = 0; i < entryCount; i++) { + String key = randomAlphaOfLength(5); + String value = randomAlphaOfLength(10); + extras.put(key, value); + } + } + return new AnalyzeResponse.AnalyzeToken(token, position, startOffset, endOffset, posLength, type, extras); + } } From fc936ee50c58121a6cfe825f52deae259c0583c6 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 28 Jun 2018 11:12:04 +0100 Subject: [PATCH 11/17] checkstyle --- .../action/admin/indices/analyze/AnalyzeResponse.java | 1 - .../action/admin/indices/analyze/AnalyzeResponseTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index 0fe42a472cd0a..a982fbb767f0f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index dab88ae3fc124..874ad649c8b8c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.Map; import java.util.function.Predicate; From ba2cec1ba0bddc8db17501d24798ea3346782e94 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 28 Jun 2018 13:14:28 +0100 Subject: [PATCH 12/17] fixes --- .../IndicesClientDocumentationIT.java | 23 ++++++++++--------- .../high-level/indices/analyze.asciidoc | 15 ++++++++---- .../indices/analyze/AnalyzeResponse.java | 5 +++- .../indices/analyze/AnalyzeResponseTests.java | 3 ++- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 0060ef9cf7ca6..55743162bae15 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2258,13 +2258,14 @@ public void testAnalyze() throws IOException, InterruptedException { AnalyzeResponse response = client.indices().analyze(request, RequestOptions.DEFAULT); // end::analyze-request-sync - // tag::analyze-response + // tag::analyze-response-tokens List tokens = response.getTokens(); // <1> + // end::analyze-response-tokens + // tag::analyze-response-detail DetailAnalyzeResponse detail = response.detail(); // <2> - // end::analyze-response + // end::analyze-response-detail - assertEquals(tokens.size(), 1); - assertEquals(tokens.get(0).getTerm(), "bar"); + assertNull(tokens); assertNotNull(detail.tokenizer()); } @@ -2274,6 +2275,7 @@ public void testAnalyze() throws IOException, InterruptedException { PutMappingRequest pmReq = new PutMappingRequest() .indices("my_index") + .type("_doc") .source("my_field", "type=text,analyzer=english"); PutMappingResponse pmResp = client.indices().putMapping(pmReq, RequestOptions.DEFAULT); assertTrue(pmResp.isAcknowledged()); @@ -2300,15 +2302,14 @@ public void onFailure(Exception e) { }; // end::analyze-execute-listener + // use a built-in analyzer in the test + request = new AnalyzeRequest(); + request.index("my_index"); + request.field("my_field"); + request.text("some text to analyze"); // Use a blocking listener in the test final CountDownLatch latch = new CountDownLatch(1); - final ActionListener blockingListener = new LatchedActionListener<>(listener, latch); - listener = ActionListener.wrap(r -> { - assertThat(r.getTokens(), hasSize(4)); - }, e-> { - blockingListener.onFailure(e); - fail("should not fail"); - }); + listener = new LatchedActionListener<>(listener, latch); // tag::analyze-request-async client.indices().analyzeAsync(request, RequestOptions.DEFAULT, listener); diff --git a/docs/java-rest/high-level/indices/analyze.asciidoc b/docs/java-rest/high-level/indices/analyze.asciidoc index ba18ef13e4451..4bffe2f020382 100644 --- a/docs/java-rest/high-level/indices/analyze.asciidoc +++ b/docs/java-rest/high-level/indices/analyze.asciidoc @@ -104,9 +104,16 @@ The returned `AnalyzeResponse` allows you to retrieve details of the analysis as follows: ["source","java",subs="attributes,callouts,macros"] --------------------------------------------------- -include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-response] +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-response-tokens] --------------------------------------------------- <1> `AnalyzeToken` holds information about the individual tokens produced by analysis -<2> `DetailAnalyzeResponse` holds more detailed information about tokens produced by -the various substeps in the analysis chain. If `explain` was set to `false` in the -request, this method will return `null` \ No newline at end of file + +If `explain` was set to `true`, then information is instead returned from the `detail()` +method: + +["source","java",subs="attributes,callouts,macros"] +--------------------------------------------------- +include-tagged::{doc-tests}/IndicesClientDocumentationIT.java[analyze-response-detail] +--------------------------------------------------- +<1> `DetailAnalyzeResponse` holds more detailed information about tokens produced by +the various substeps in the analysis chain. \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index a982fbb767f0f..e1a76a2a7bc88 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -274,7 +274,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("analyze_response", true, args -> new AnalyzeResponse((List) args[0], (DetailAnalyzeResponse) args[1])); static { - PARSER.declareObjectArray(constructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS)); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> AnalyzeToken.fromXContent(p), new ParseField(Fields.TOKENS)); PARSER.declareObject(optionalConstructorArg(), DetailAnalyzeResponse.PARSER, new ParseField(Fields.DETAIL)); } @@ -290,6 +290,9 @@ public void readFrom(StreamInput in) throws IOException { for (int i = 0; i < size; i++) { tokens.add(AnalyzeToken.readAnalyzeToken(in)); } + if (tokens.size() == 0) { + tokens = null; + } detail = in.readOptionalStreamable(DetailAnalyzeResponse::new); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 874ad649c8b8c..4c2173945867e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -67,8 +67,9 @@ protected AnalyzeResponse createTestInstance() { new DetailAnalyzeResponse.AnalyzeTokenList("my_tokenfilter_2", tokens) }); } + return new AnalyzeResponse(null, dar); } - return new AnalyzeResponse(Arrays.asList(tokens), dar); + return new AnalyzeResponse(Arrays.asList(tokens), null); } private AnalyzeResponse.AnalyzeToken randomToken() { From d573dbb4a049d1a9e33153696bc2b3474ea55f68 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 28 Jun 2018 21:40:07 +0100 Subject: [PATCH 13/17] checkstyle --- .../action/admin/indices/analyze/AnalyzeResponse.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index e1a76a2a7bc88..b2c3b69f02810 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.TreeMap; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; From 070c3cb719f2f68f70e3b4cae0b9dfd9bbf92ff7 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 29 Jun 2018 12:33:27 +0100 Subject: [PATCH 14/17] Deal with arbitrary objects as attributes --- .../indices/analyze/AnalyzeResponse.java | 7 ++++- .../indices/analyze/AnalyzeResponseTests.java | 26 ++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index b2c3b69f02810..d3840fb145608 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -184,7 +184,12 @@ else if (t == XContentParser.Token.VALUE_NUMBER) { else if (t == XContentParser.Token.VALUE_BOOLEAN) { attributes.put(field, parser.booleanValue()); } - // TODO: Can we parse arbitrary objects into maps? + else if (t == XContentParser.Token.START_OBJECT) { + attributes.put(field, parser.map()); + } + else if (t == XContentParser.Token.START_ARRAY) { + attributes.put(field, parser.list()); + } } } return new AnalyzeToken(term, position, startOffset, endOffset, positionLength, type, attributes); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 4c2173945867e..7656e21bed3f6 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -23,8 +23,10 @@ import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -83,9 +85,27 @@ private AnalyzeResponse.AnalyzeToken randomToken() { if (randomBoolean()) { int entryCount = randomInt(6); for (int i = 0; i < entryCount; i++) { - String key = randomAlphaOfLength(5); - String value = randomAlphaOfLength(10); - extras.put(key, value); + switch (randomInt(6)) { + case 0: + case 1: + case 2: + case 3: + String key = randomAlphaOfLength(5); + String value = randomAlphaOfLength(10); + extras.put(key, value); + break; + case 4: + String objkey = randomAlphaOfLength(5); + Map obj = new HashMap<>(); + obj.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); + extras.put(objkey, obj); + case 5: + String listkey = randomAlphaOfLength(5); + List list = new ArrayList<>(); + list.add(randomAlphaOfLength(4)); + list.add(randomAlphaOfLength(6)); + extras.put(listkey, list); + } } } return new AnalyzeResponse.AnalyzeToken(token, position, startOffset, endOffset, posLength, type, extras); From 08d27668811dcde66e7ff67f7e7037fac95a3746 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Fri, 29 Jun 2018 16:36:40 +0100 Subject: [PATCH 15/17] compilation, apparently --- .../action/admin/indices/analyze/AnalyzeResponseTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java index 7656e21bed3f6..404db74a46e12 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponseTests.java @@ -99,12 +99,14 @@ private AnalyzeResponse.AnalyzeToken randomToken() { Map obj = new HashMap<>(); obj.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); extras.put(objkey, obj); + break; case 5: String listkey = randomAlphaOfLength(5); List list = new ArrayList<>(); list.add(randomAlphaOfLength(4)); list.add(randomAlphaOfLength(6)); extras.put(listkey, list); + break; } } } From 0e7ce0115560aa67b08f01777b0fb18ee9c45095 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 3 Jul 2018 10:25:21 +0100 Subject: [PATCH 16/17] feedback --- .../indices/analyze/AnalyzeResponse.java | 30 ++++-------- .../analyze/DetailAnalyzeResponse.java | 46 +++++++++---------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java index d3840fb145608..d45ab2682a5ec 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java @@ -158,36 +158,26 @@ public static AnalyzeToken fromXContent(XContentParser parser) throws IOExceptio } if (Fields.TOKEN.equals(field)) { term = parser.text(); - } - else if (Fields.POSITION.equals(field)) { + } else if (Fields.POSITION.equals(field)) { position = parser.intValue(); - } - else if (Fields.START_OFFSET.equals(field)) { + } else if (Fields.START_OFFSET.equals(field)) { startOffset = parser.intValue(); - } - else if (Fields.END_OFFSET.equals(field)) { + } else if (Fields.END_OFFSET.equals(field)) { endOffset = parser.intValue(); - } - else if (Fields.POSITION_LENGTH.equals(field)) { + } else if (Fields.POSITION_LENGTH.equals(field)) { positionLength = parser.intValue(); - } - else if (Fields.TYPE.equals(field)) { + } else if (Fields.TYPE.equals(field)) { type = parser.text(); - } - else { + } else { if (t == XContentParser.Token.VALUE_STRING) { attributes.put(field, parser.text()); - } - else if (t == XContentParser.Token.VALUE_NUMBER) { + } else if (t == XContentParser.Token.VALUE_NUMBER) { attributes.put(field, parser.numberValue()); - } - else if (t == XContentParser.Token.VALUE_BOOLEAN) { + } else if (t == XContentParser.Token.VALUE_BOOLEAN) { attributes.put(field, parser.booleanValue()); - } - else if (t == XContentParser.Token.START_OBJECT) { + } else if (t == XContentParser.Token.START_OBJECT) { attributes.put(field, parser.map()); - } - else if (t == XContentParser.Token.START_ARRAY) { + } else if (t == XContentParser.Token.START_ARRAY) { attributes.put(field, parser.list()); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java index b9c305596ae6f..1e0c4ed525ef1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/DetailAnalyzeResponse.java @@ -42,35 +42,15 @@ public class DetailAnalyzeResponse implements Streamable, ToXContentFragment { - DetailAnalyzeResponse() { - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DetailAnalyzeResponse that = (DetailAnalyzeResponse) o; - return customAnalyzer == that.customAnalyzer && - Objects.equals(analyzer, that.analyzer) && - Arrays.equals(charfilters, that.charfilters) && - Objects.equals(tokenizer, that.tokenizer) && - Arrays.equals(tokenfilters, that.tokenfilters); - } - - @Override - public int hashCode() { - int result = Objects.hash(customAnalyzer, analyzer, tokenizer); - result = 31 * result + Arrays.hashCode(charfilters); - result = 31 * result + Arrays.hashCode(tokenfilters); - return result; - } - private boolean customAnalyzer = false; private AnalyzeTokenList analyzer; private CharFilteredText[] charfilters; private AnalyzeTokenList tokenizer; private AnalyzeTokenList[] tokenfilters; + DetailAnalyzeResponse() { + } + public DetailAnalyzeResponse(AnalyzeTokenList analyzer) { this(false, analyzer, null, null, null); } @@ -131,6 +111,26 @@ public DetailAnalyzeResponse tokenfilters(AnalyzeTokenList[] tokenfilters) { return this; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DetailAnalyzeResponse that = (DetailAnalyzeResponse) o; + return customAnalyzer == that.customAnalyzer && + Objects.equals(analyzer, that.analyzer) && + Arrays.equals(charfilters, that.charfilters) && + Objects.equals(tokenizer, that.tokenizer) && + Arrays.equals(tokenfilters, that.tokenfilters); + } + + @Override + public int hashCode() { + int result = Objects.hash(customAnalyzer, analyzer, tokenizer); + result = 31 * result + Arrays.hashCode(charfilters); + result = 31 * result + Arrays.hashCode(tokenfilters); + return result; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.CUSTOM_ANALYZER, customAnalyzer); From 20fc870138eb1ac78c53e1c76c2ae7b3cd842077 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 3 Jul 2018 13:46:21 +0100 Subject: [PATCH 17/17] docs fixes --- .../client/documentation/IndicesClientDocumentationIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index d4bc241b25b44..964757db372ae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -2356,8 +2356,8 @@ public void testAnalyze() throws IOException, InterruptedException { // end::analyze-custom-normalizer-request // tag::analyze-request-explain - request.explain(true); - request.attributes("keyword", "type"); + request.explain(true); // <1> + request.attributes("keyword", "type"); // <2> // end::analyze-request-explain // tag::analyze-request-sync @@ -2368,7 +2368,7 @@ public void testAnalyze() throws IOException, InterruptedException { List tokens = response.getTokens(); // <1> // end::analyze-response-tokens // tag::analyze-response-detail - DetailAnalyzeResponse detail = response.detail(); // <2> + DetailAnalyzeResponse detail = response.detail(); // <1> // end::analyze-response-detail assertNull(tokens);