From 64140c3783041bcb5da62961030042ab5321cbd7 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 19 Apr 2018 16:30:00 -0700 Subject: [PATCH 1/9] Add support for field capabilities to the high-level REST client. --- .../org/elasticsearch/client/Request.java | 19 +++ .../client/RestHighLevelClient.java | 27 ++++ .../client/FieldCapabilitiesIT.java | 133 ++++++++++++++++++ .../elasticsearch/client/RequestTests.java | 42 +++++- .../action/fieldcaps/FieldCapabilities.java | 53 +++++-- .../fieldcaps/FieldCapabilitiesRequest.java | 8 +- .../fieldcaps/FieldCapabilitiesResponse.java | 43 +++++- .../FieldCapabilitiesResponseTests.java | 116 +++++++++++++++ 8 files changed, 428 insertions(+), 13 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java index 500130ed39705..d68d3b309af51 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java @@ -48,6 +48,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -75,6 +76,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.rankeval.RankEvalRequest; +import org.elasticsearch.rest.action.RestFieldCapabilitiesAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -536,6 +538,16 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) { return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null); } + static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) { + Params params = Params.builder(); + params.withFields(fieldCapabilitiesRequest.fields()); + params.withIndicesOptions(fieldCapabilitiesRequest.indicesOptions()); + + String[] indices = fieldCapabilitiesRequest.indices(); + String endpoint = endpoint(indices, "_field_caps"); + return new Request(HttpGet.METHOD_NAME, endpoint, params.getParams(), null); + } + static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException { String endpoint = endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval"); Params params = Params.builder(); @@ -712,6 +724,13 @@ Params withFetchSourceContext(FetchSourceContext fetchSourceContext) { return this; } + Params withFields(String[] fields) { + if (fields != null && fields.length > 0) { + return putParam("fields", String.join(",", fields)); + } + return this; + } + Params withMasterTimeout(TimeValue masterTimeout) { return putParam("master_timeout", masterTimeout); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index bf80aa7720741..c6d5e947f2c62 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -30,6 +30,8 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetRequest; @@ -501,6 +503,31 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener< headers); } + /** + * Executes a request using the Field Capabilities API. + * + * See Field Capabilities API + * on elastic.co. + */ + public final FieldCapabilitiesResponse fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest, + Header... headers) throws IOException { + return performRequestAndParseEntity(fieldCapabilitiesRequest, Request::fieldCaps, + FieldCapabilitiesResponse::fromXContent, emptySet(), headers); + } + + /** + * Asynchronously executes a request using the Field Capabilities API. + * + * See Field Capabilities API + * on elastic.co. + */ + public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesRequest, + ActionListener listener, + Header... headers) { + performRequestAsyncAndParseEntity(fieldCapabilitiesRequest, Request::fieldCaps, + FieldCapabilitiesResponse::fromXContent, listener, emptySet(), headers); + } + protected final Resp performRequestAndParseEntity(Req request, CheckedFunction requestConverter, CheckedFunction entityParser, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java new file mode 100644 index 0000000000000..d7932d55088f0 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.rest.RestStatus; +import org.junit.Before; +import org.junit.Ignore; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.Matchers.hasKey; + +public class FieldCapabilitiesIT extends ESRestHighLevelClientTestCase { + @Before + public void indexDocuments() throws Exception { + StringEntity document1 = new StringEntity( + "{" + + "\"rating\": 7," + + "\"title\": \"first title\"" + + "}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/index1/doc/1", Collections.emptyMap(), document1); + + StringEntity mappings = new StringEntity( + "{" + + " \"mappings\": {" + + " \"doc\": {" + + " \"properties\": {" + + " \"rating\": {" + + " \"type\": \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + "}}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/" + "index2", Collections.emptyMap(), mappings); + + StringEntity document2 = new StringEntity( + "{" + + "\"rating\": \"good\"," + + "\"title\": \"second title\"" + + "}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/index2/doc/1", Collections.emptyMap(), document2); + + client().performRequest("POST", "/_refresh"); + } + + public void testBasicRequest() throws IOException { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index1", "index2") + .fields("rating", "title"); + + FieldCapabilitiesResponse response = execute(request, + highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); + + // Check the capabilities for the 'rating' field. + assertTrue(response.get().containsKey("rating")); + Map ratingResponse = response.getField("rating"); + assertEquals(2, ratingResponse.size()); + + FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( + "rating", "keyword", true, true, new String[]{"index2"}, null, null); + assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); + + FieldCapabilities expectedLongCapabilities = new FieldCapabilities( + "rating", "long", true, true, new String[]{"index1"}, null, null); + assertEquals(expectedLongCapabilities, ratingResponse.get("long")); + + // Check the capabilities for the 'title' field. + assertTrue(response.get().containsKey("title")); + Map titleResponse = response.getField("title"); + assertEquals(1, titleResponse.size()); + + FieldCapabilities expectedTextCapabilities = new FieldCapabilities( + "title", "text", true, false); + assertEquals(expectedTextCapabilities, titleResponse.get("text")); + } + + public void testNonexistentFields() throws IOException { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index2") + .fields("nonexistent"); + + FieldCapabilitiesResponse response = execute(request, + highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); + assertTrue(response.get().isEmpty()); + } + + public void testNoFields() { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index2"); + + expectThrows(IllegalArgumentException.class, + () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync)); + } + + public void testNonexistentIndices() { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("nonexistent") + .fields("rating"); + + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync)); + assertEquals(RestStatus.NOT_FOUND, exception.status()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java index f691c60daa5da..f24014a598ff2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; @@ -89,6 +90,7 @@ import org.elasticsearch.index.rankeval.RankEvalSpec; import org.elasticsearch.index.rankeval.RatedRequest; import org.elasticsearch.index.rankeval.RestRankEvalAction; +import org.elasticsearch.rest.action.RestFieldCapabilitiesAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; @@ -128,6 +130,8 @@ import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.nullValue; public class RequestTests extends ESTestCase { @@ -1213,6 +1217,43 @@ public void testExistsAliasNoAliasNoIndex() { } } + public void testFieldCaps() { + // Create a random request. + String[] indices = randomIndicesNames(0, 5); + String[] fields = generateRandomStringArray(5, 10, false, false); + + FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest() + .indices(indices) + .fields(fields); + + Map expectedIndicesParams = new HashMap<>(); + setRandomIndicesOptions(fieldCapabilitiesRequest::indicesOptions, + fieldCapabilitiesRequest::indicesOptions, + expectedIndicesParams); + + Request request = Request.fieldCaps(fieldCapabilitiesRequest); + + // Verify that the resulting REST request looks as expected. + StringJoiner expectedEndpoint = new StringJoiner("/", "/", ""); + String joinedIndices = String.join(",", indices); + if (!joinedIndices.isEmpty()) { + expectedEndpoint.add(joinedIndices); + } + expectedEndpoint.add("_field_caps"); + + assertEquals(expectedEndpoint.toString(), request.getEndpoint()); + assertEquals(4, request.getParameters().size()); + + // Note that we don't check the field param value explicitly, as field + // names are added to the request in a non-deterministic order. + assertThat(request.getParameters(), hasKey("fields")); + for (Map.Entry param : expectedIndicesParams.entrySet()) { + assertThat(request.getParameters(), hasEntry(param.getKey(), param.getValue())); + } + + assertNull(request.getEntity()); + } + public void testRankEval() throws Exception { RankEvalSpec spec = new RankEvalSpec( Collections.singletonList(new RatedRequest("queryId", Collections.emptyList(), new SearchSourceBuilder())), @@ -1233,7 +1274,6 @@ public void testRankEval() throws Exception { assertEquals(3, request.getParameters().size()); assertEquals(expectedParams, request.getParameters()); assertToXContentBody(spec, request.getEntity()); - } public void testSplit() throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java index ec6d0902ac98a..21bb452430e7a 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; @@ -36,6 +39,13 @@ * Describes the capabilities of a field optionally merged across multiple indices. */ public class FieldCapabilities implements Writeable, ToXContentObject { + private static final ParseField TYPE_FIELD = new ParseField("type"); + private static final ParseField SEARCHABLE_FIELD = new ParseField("searchable"); + private static final ParseField AGGREGATABLE_FIELD = new ParseField("aggregatable"); + private static final ParseField INDICES_FIELD = new ParseField("indices"); + private static final ParseField NON_SEARCHABLE_INDICES_FIELD = new ParseField("non_searchable_indices"); + private static final ParseField NON_AGGREGATABLE_INDICES_FIELD = new ParseField("non_aggregatable_indices"); + private final String name; private final String type; private final boolean isSearchable; @@ -52,7 +62,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param isSearchable Whether this field is indexed for search. * @param isAggregatable Whether this field can be aggregated on. */ - FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { + public FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { this(name, type, isSearchable, isAggregatable, null, null, null); } @@ -69,7 +79,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { * @param nonAggregatableIndices The list of indices where this field is not aggregatable, * or null if the field is aggregatable in all indices. */ - FieldCapabilities(String name, String type, + public FieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable, String[] indices, String[] nonSearchableIndices, @@ -83,7 +93,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject { this.nonAggregatableIndices = nonAggregatableIndices; } - FieldCapabilities(StreamInput in) throws IOException { + public FieldCapabilities(StreamInput in) throws IOException { this.name = in.readString(); this.type = in.readString(); this.isSearchable = in.readBoolean(); @@ -107,22 +117,47 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("type", type); - builder.field("searchable", isSearchable); - builder.field("aggregatable", isAggregatable); + builder.field(TYPE_FIELD.getPreferredName(), type); + builder.field(SEARCHABLE_FIELD.getPreferredName(), isSearchable); + builder.field(AGGREGATABLE_FIELD.getPreferredName(), isAggregatable); if (indices != null) { - builder.field("indices", indices); + builder.field(INDICES_FIELD.getPreferredName(), indices); } if (nonSearchableIndices != null) { - builder.field("non_searchable_indices", nonSearchableIndices); + builder.field(NON_SEARCHABLE_INDICES_FIELD.getPreferredName(), nonSearchableIndices); } if (nonAggregatableIndices != null) { - builder.field("non_aggregatable_indices", nonAggregatableIndices); + builder.field(NON_AGGREGATABLE_INDICES_FIELD.getPreferredName(), nonAggregatableIndices); } builder.endObject(); return builder; } + public static FieldCapabilities fromXContent(String name, XContentParser parser) throws IOException { + return PARSER.parse(parser, name); + } + + @SuppressWarnings("unchecked") + private static ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "field_capabilities", + true, + (a, name) -> new FieldCapabilities(name, + (String) a[0], + (boolean) a[1], + (boolean) a[2], + a[3] != null ? ((List) a[3]).toArray(new String[0]) : null, + a[4] != null ? ((List) a[4]).toArray(new String[0]) : null, + a[5] != null ? ((List) a[5]).toArray(new String[0]) : null)); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), SEARCHABLE_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), AGGREGATABLE_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_SEARCHABLE_INDICES_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), NON_AGGREGATABLE_INDICES_FIELD); + } + /** * The name of the field. */ diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index b04f882076326..264fa21cf9188 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -61,14 +61,18 @@ public FieldCapabilitiesRequest() {} /** * Returns true iff the results should be merged. + * + * Note that when using the high-level REST client, results are always merged (this flag is always considered 'true'). */ boolean isMergeResults() { return mergeResults; } /** - * if set to true the response will contain only a merged view of the per index field capabilities. Otherwise only - * unmerged per index field capabilities are returned. + * If set to true the response will contain only a merged view of the per index field capabilities. + * Otherwise only unmerged per index field capabilities are returned. + * + * Note that when using the high-level REST client, results are always merged (this flag is always considered 'true'). */ void setMergeResults(boolean mergeResults) { this.mergeResults = mergeResults; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 4b1bcf575899f..5e2202ac073af 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -21,20 +21,29 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Response for {@link FieldCapabilitiesRequest} requests. */ public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentFragment { + private static final ParseField FIELDS_FIELD = new ParseField("fields"); + private Map> responseMap; private List indexResponses; @@ -114,10 +123,42 @@ private static void writeField(StreamOutput out, @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("fields", responseMap); + builder.field(FIELDS_FIELD.getPreferredName(), responseMap); return builder; } + public static FieldCapabilitiesResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("field_capabilities_response", true, + a -> new FieldCapabilitiesResponse( + ((List>>) a[0]).stream() + .collect(Collectors.toMap(Tuple::v1, Tuple::v2)))); + + static { + PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> { + Map typeToCapabilities = parseTypeToCapabilities(p, n); + return new Tuple<>(n, typeToCapabilities); + }, FIELDS_FIELD); + } + + private static Map parseTypeToCapabilities(XContentParser parser, String name) throws IOException { + Map typeToCapabilities = new HashMap<>(); + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + String type = parser.currentName(); + FieldCapabilities capabilities = FieldCapabilities.fromXContent(name, parser); + typeToCapabilities.put(type, capabilities); + } + return typeToCapabilities; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 2eaf1d4832f3f..adaad7c242792 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -19,13 +19,22 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; + +import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; public class FieldCapabilitiesResponseTests extends ESTestCase { private FieldCapabilitiesResponse randomResponse() { @@ -57,4 +66,111 @@ public void testSerialization() throws IOException { assertEquals(deserialized.hashCode(), response.hashCode()); } } + + public void testToXContent() throws IOException { + FieldCapabilitiesResponse response = createSimpleResponse(); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON) + .startObject(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + String generatedResponse = BytesReference.bytes(builder).utf8ToString(); + assertEquals(( + "{" + + " \"fields\": {" + + " \"rating\": { " + + " \"keyword\": {" + + " \"type\": \"keyword\"," + + " \"searchable\": false," + + " \"aggregatable\": true," + + " \"indices\": [\"index3\", \"index4\"]," + + " \"non_searchable_indices\": [\"index4\"] " + + " }," + + " \"long\": {" + + " \"type\": \"long\"," + + " \"searchable\": true," + + " \"aggregatable\": false," + + " \"indices\": [\"index1\", \"index2\"]," + + " \"non_aggregatable_indices\": [\"index1\"] " + + " }" + + " }," + + " \"title\": { " + + " \"text\": {" + + " \"type\": \"text\"," + + " \"searchable\": true," + + " \"aggregatable\": false" + + " }" + + " }" + + " }" + + "}").replaceAll("\\s+", ""), generatedResponse); + } + + private static FieldCapabilitiesResponse createSimpleResponse() { + Map titleCapabilities = new HashMap<>(); + titleCapabilities.put("text", new FieldCapabilities("title", "text", true, false)); + + Map ratingCapabilities = new HashMap<>(); + ratingCapabilities.put("long", new FieldCapabilities("rating", "long", + true, false, + new String[]{"index1", "index2"}, + null, + new String[]{"index1"})); + ratingCapabilities.put("keyword", new FieldCapabilities("rating", "keyword", + false, true, + new String[]{"index3", "index4"}, + new String[]{"index4"}, + null)); + + Map> responses = new HashMap<>(); + responses.put("title", titleCapabilities); + responses.put("rating", ratingCapabilities); + return new FieldCapabilitiesResponse(responses); + } + + public void testFromXContent() throws IOException { + FieldCapabilitiesResponse response = createRandomResponse(); + boolean humanReadable = randomBoolean(); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference content = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + + Predicate excludedPaths = path -> path.startsWith("fields"); + BytesReference contentWithRandomFields = insertRandomFields(xContentType, content, excludedPaths, random()); + + FieldCapabilitiesResponse parsedResponse; + try (XContentParser parser = createParser(xContentType.xContent(), contentWithRandomFields)) { + parsedResponse = FieldCapabilitiesResponse.fromXContent(parser); + assertNull(parser.nextToken()); + } + + assertNotSame(response, parsedResponse); + assertEquals(response, parsedResponse); + } + + private static FieldCapabilitiesResponse createRandomResponse() { + Map> responses = new HashMap<>(); + + String[] fields = generateRandomStringArray(5, 10, false, true); + assertNotNull(fields); + + for (String field : fields) { + responses.put(field, new HashMap<>()); + + String[] types = generateRandomStringArray(5, 10, false, false); + assertNotNull(types); + + for (String type : types) { + FieldCapabilities capabilities = new FieldCapabilities(field, type, + randomBoolean(), + randomBoolean(), + generateRandomStringArray(5, 10, true, false), + generateRandomStringArray(3, 10, true, false), + generateRandomStringArray(3, 10, true, false)); + + Map typesToCapabilities = responses.get(field); + typesToCapabilities.put(type, capabilities); + } + } + return new FieldCapabilitiesResponse(responses); + } } From 800b356894d173d66b45eede932908a710dca2a6 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Mon, 23 Apr 2018 14:13:21 -0700 Subject: [PATCH 2/9] Add documentation for field capabilities in the high-level REST API. --- .../documentation/SearchDocumentationIT.java | 93 +++++++++++++++++-- .../high-level/search/field-caps.asciidoc | 72 ++++++++++++++ .../high-level/supported-apis.asciidoc | 2 + 3 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 docs/java-rest/high-level/search/field-caps.asciidoc diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 52f6984e65107..bf6fb6fd5de07 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -21,8 +21,13 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.ClearScrollRequest; @@ -93,6 +98,8 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -699,6 +706,61 @@ public void onFailure(Exception e) { } } + public void testFieldCaps() throws Exception { + indexSearchTestData(); + RestHighLevelClient client = highLevelClient(); + // tag::field-caps-request + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .fields("user") + .indices("posts", "authors", "contributors"); + // end::field-caps-request + + // tag::field-caps-execute + FieldCapabilitiesResponse response = client.fieldCaps(request); + // end::field-caps-execute + + // tag::field-caps-response + assertThat(response.get().keySet(), contains("user")); + Map userResponse = response.getField("user"); + + assertThat(userResponse.keySet(), containsInAnyOrder("keyword", "text")); // <1> + FieldCapabilities textCapabilities = userResponse.get("keyword"); + + assertTrue(textCapabilities.isSearchable()); + assertFalse(textCapabilities.isAggregatable()); + + assertArrayEquals(textCapabilities.indices(), // <2> + new String[]{"authors", "contributors"}); + assertNull(textCapabilities.nonSearchableIndices()); // <3> + assertArrayEquals(textCapabilities.nonAggregatableIndices(), // <4> + new String[]{"authors"}); + // end::field-caps-response + + // tag::field-caps-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(FieldCapabilitiesResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::field-caps-execute-listener + + // Replace the empty listener by a blocking listener for tests. + CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::field-caps-execute-async + client.fieldCapsAsync(request, listener); // <1> + // end::field-caps-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + public void testRankEval() throws Exception { indexSearchTestData(); RestHighLevelClient client = highLevelClient(); @@ -794,7 +856,7 @@ public void testMultiSearch() throws Exception { MultiSearchResponse.Item firstResponse = response.getResponses()[0]; // <1> assertNull(firstResponse.getFailure()); // <2> SearchResponse searchResponse = firstResponse.getResponse(); // <3> - assertEquals(3, searchResponse.getHits().getTotalHits()); + assertEquals(4, searchResponse.getHits().getTotalHits()); MultiSearchResponse.Item secondResponse = response.getResponses()[1]; // <4> assertNull(secondResponse.getFailure()); searchResponse = secondResponse.getResponse(); @@ -840,18 +902,35 @@ public void onFailure(Exception e) { } private void indexSearchTestData() throws IOException { - BulkRequest request = new BulkRequest(); - request.add(new IndexRequest("posts", "doc", "1") + CreateIndexRequest authorsRequest = new CreateIndexRequest("authors") + .mapping("doc", "user", "type=keyword,doc_values=false"); + CreateIndexResponse authorsResponse = highLevelClient().indices().create(authorsRequest); + assertTrue(authorsResponse.isAcknowledged()); + + CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors") + .mapping("doc", "user", "type=keyword"); + CreateIndexResponse reviewersResponse = highLevelClient().indices().create(reviewersRequest); + assertTrue(reviewersResponse.isAcknowledged()); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("posts", "doc", "1") .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user", Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value"))); - request.add(new IndexRequest("posts", "doc", "2") + bulkRequest.add(new IndexRequest("posts", "doc", "2") .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user", Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value"))); - request.add(new IndexRequest("posts", "doc", "3") + bulkRequest.add(new IndexRequest("posts", "doc", "3") .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user", Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value"))); - request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - BulkResponse bulkResponse = highLevelClient().bulk(request); + + bulkRequest.add(new IndexRequest("authors", "doc", "1") + .source(XContentType.JSON, "user", "kimchy")); + bulkRequest.add(new IndexRequest("contributors", "doc", "1") + .source(XContentType.JSON, "user", "tanguy")); + + + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest); assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } diff --git a/docs/java-rest/high-level/search/field-caps.asciidoc b/docs/java-rest/high-level/search/field-caps.asciidoc new file mode 100644 index 0000000000000..75cd85dfbc3fc --- /dev/null +++ b/docs/java-rest/high-level/search/field-caps.asciidoc @@ -0,0 +1,72 @@ +[[java-rest-high-field-caps]] +=== Field Capabilities API + +The field capabilities API allows for retrieving the capabilities of fields across multiple indices. + +[[java-rest-high-field-caps-request]] +==== Field Capabilities Request + +A `FieldCapabilitiesRequest` contains a list of fields to get capabilities for, +should be returned, plus an optional list of target indices. If no indices +are provided, the request will be executed on all indices. + +Note that fields parameter supports wildcard notation. For example, providing `text_*` +will cause all fields that match the expression to be returned. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-request] +-------------------------------------------------- + +[[java-rest-high-field-caps-sync]] +==== Synchronous Execution + +The `fieldCaps` method executes the request synchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-execute] +-------------------------------------------------- + +[[java-rest-high-field-caps-async]] +==== Asynchronous Execution + +The `fieldCapsAsync` method executes the request asynchronously, +calling the provided `ActionListener` when the response is ready: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-execute-async] +-------------------------------------------------- +<1> The `FieldCapabilitiesRequest` to execute and the `ActionListener` to use when +the execution completes. + +The asynchronous method does not block and returns immediately. Once the request +completes, the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `FieldCapabilitiesResponse` is constructed as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. +<2> Called when the whole `FieldCapabilitiesRequest` fails. + +[[java-rest-high-field-caps-response]] +==== FieldCapabilitiesResponse + +For each requested field, the returned `FieldCapabilitiesResponse` contains its type +and whether or not it can be searched or aggregated on. The response also gives +information about how each index contributes to the field's capabilities. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-response] +-------------------------------------------------- +<1> The `user` field has two possible types, `keyword` and `text`. +<2> This field only has type `keyword` in the `authors` and `contributors` indices. +<3> Null, since the field is searchable in all indices for which it has the `keyword` type. +<4> The `user` field is not aggregatable in the `authors` index. \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 1f3d7a3744300..1c0e09c6c079e 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -32,11 +32,13 @@ The Java High Level REST Client supports the following Search APIs: * <> * <> * <> +* <> * <> include::search/search.asciidoc[] include::search/scroll.asciidoc[] include::search/multi-search.asciidoc[] +include::search/field-caps.asciidoc[] include::search/rank-eval.asciidoc[] == Miscellaneous APIs From a99839707390d016e3b25864427c04961b0b416b Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 12:51:52 -0700 Subject: [PATCH 3/9] Fix failures in SearchDocumentationIT introduced when adding field capabilities tests. --- .../client/documentation/SearchDocumentationIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index bf6fb6fd5de07..12be190dc33f0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -164,6 +164,7 @@ public void testSearch() throws Exception { // tag::search-source-setter SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("posts"); searchRequest.source(sourceBuilder); // end::search-source-setter From eb2eade0c1c64a2d3f21918c60f058c81bc5e7bc Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 13:24:04 -0700 Subject: [PATCH 4/9] Move the new cases in FieldCapabilitiesIT into existing test classes. --- .../client/FieldCapabilitiesIT.java | 133 ------------------ .../org/elasticsearch/client/SearchIT.java | 75 +++++++++- .../FieldCapabilitiesRequestTests.java | 11 +- 3 files changed, 83 insertions(+), 136 deletions(-) delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java deleted file mode 100644 index d7932d55088f0..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/FieldCapabilitiesIT.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.fieldcaps.FieldCapabilities; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; -import org.elasticsearch.rest.RestStatus; -import org.junit.Before; -import org.junit.Ignore; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import static org.hamcrest.Matchers.hasKey; - -public class FieldCapabilitiesIT extends ESRestHighLevelClientTestCase { - @Before - public void indexDocuments() throws Exception { - StringEntity document1 = new StringEntity( - "{" + - "\"rating\": 7," + - "\"title\": \"first title\"" + - "}", - ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/index1/doc/1", Collections.emptyMap(), document1); - - StringEntity mappings = new StringEntity( - "{" + - " \"mappings\": {" + - " \"doc\": {" + - " \"properties\": {" + - " \"rating\": {" + - " \"type\": \"keyword\"" + - " }" + - " }" + - " }" + - " }" + - "}}", - ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/" + "index2", Collections.emptyMap(), mappings); - - StringEntity document2 = new StringEntity( - "{" + - "\"rating\": \"good\"," + - "\"title\": \"second title\"" + - "}", - ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/index2/doc/1", Collections.emptyMap(), document2); - - client().performRequest("POST", "/_refresh"); - } - - public void testBasicRequest() throws IOException { - FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() - .indices("index1", "index2") - .fields("rating", "title"); - - FieldCapabilitiesResponse response = execute(request, - highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); - - // Check the capabilities for the 'rating' field. - assertTrue(response.get().containsKey("rating")); - Map ratingResponse = response.getField("rating"); - assertEquals(2, ratingResponse.size()); - - FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", true, true, new String[]{"index2"}, null, null); - assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); - - FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", true, true, new String[]{"index1"}, null, null); - assertEquals(expectedLongCapabilities, ratingResponse.get("long")); - - // Check the capabilities for the 'title' field. - assertTrue(response.get().containsKey("title")); - Map titleResponse = response.getField("title"); - assertEquals(1, titleResponse.size()); - - FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "title", "text", true, false); - assertEquals(expectedTextCapabilities, titleResponse.get("text")); - } - - public void testNonexistentFields() throws IOException { - FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() - .indices("index2") - .fields("nonexistent"); - - FieldCapabilitiesResponse response = execute(request, - highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); - assertTrue(response.get().isEmpty()); - } - - public void testNoFields() { - FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() - .indices("index2"); - - expectThrows(IllegalArgumentException.class, - () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync)); - } - - public void testNonexistentIndices() { - FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() - .indices("nonexistent") - .fields("rating"); - - ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync)); - assertEquals(RestStatus.NOT_FOUND, exception.status()); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 01ef0598100fb..9828041332b32 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -27,6 +27,9 @@ import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.MultiSearchRequest; @@ -96,14 +99,31 @@ public void indexDocuments() throws IOException { client().performRequest(HttpPut.METHOD_NAME, "/index/type/5", Collections.emptyMap(), doc5); client().performRequest(HttpPost.METHOD_NAME, "/index/_refresh"); - StringEntity doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON); + + StringEntity doc = new StringEntity("{\"field\":\"value1\", \"rating\": 7}", ContentType.APPLICATION_JSON); client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/1", Collections.emptyMap(), doc); doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/2", Collections.emptyMap(), doc); - doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON); + + StringEntity mappings = new StringEntity( + "{" + + " \"mappings\": {" + + " \"doc\": {" + + " \"properties\": {" + + " \"rating\": {" + + " \"type\": \"keyword\"" + + " }" + + " }" + + " }" + + " }" + + "}}", + ContentType.APPLICATION_JSON); + client().performRequest("PUT", "/index2", Collections.emptyMap(), mappings); + doc = new StringEntity("{\"field\":\"value1\", \"rating\": \"good\"}", ContentType.APPLICATION_JSON); client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/3", Collections.emptyMap(), doc); doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/4", Collections.emptyMap(), doc); + doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON); client().performRequest(HttpPut.METHOD_NAME, "/index3/doc/5", Collections.emptyMap(), doc); doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); @@ -713,6 +733,57 @@ public void testMultiSearch_failure() throws Exception { assertThat(multiSearchResponse.getResponses()[1].getResponse(), nullValue()); } + public void testFieldCaps() throws IOException { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index1", "index2") + .fields("rating", "field"); + + FieldCapabilitiesResponse response = execute(request, + highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); + + // Check the capabilities for the 'rating' field. + assertTrue(response.get().containsKey("rating")); + Map ratingResponse = response.getField("rating"); + assertEquals(2, ratingResponse.size()); + + FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( + "rating", "keyword", true, true, new String[]{"index2"}, null, null); + assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); + + FieldCapabilities expectedLongCapabilities = new FieldCapabilities( + "rating", "long", true, true, new String[]{"index1"}, null, null); + assertEquals(expectedLongCapabilities, ratingResponse.get("long")); + + // Check the capabilities for the 'field' field. + assertTrue(response.get().containsKey("field")); + Map fieldResponse = response.getField("field"); + assertEquals(1, fieldResponse.size()); + + FieldCapabilities expectedTextCapabilities = new FieldCapabilities( + "field", "text", true, false); + assertEquals(expectedTextCapabilities, fieldResponse.get("text")); + } + + public void testFieldCapsWithNonExistentFields() throws IOException { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index2") + .fields("nonexistent"); + + FieldCapabilitiesResponse response = execute(request, + highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync); + assertTrue(response.get().isEmpty()); + } + + public void testFieldCapsWithNonExistentIndices() { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("non-existent") + .fields("rating"); + + ElasticsearchException exception = expectThrows(ElasticsearchException.class, + () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync)); + assertEquals(RestStatus.NOT_FOUND, exception.status()); + } + private static void assertSearchHeader(SearchResponse searchResponse) { assertThat(searchResponse.getTook().nanos(), greaterThanOrEqualTo(0L)); assertEquals(0, searchResponse.getFailedShards()); diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java index 8543b35569a31..9f893ada9c735 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; @@ -80,7 +82,7 @@ public void testEqualsAndHashcode() { } - public void testFieldCapsRequestSerialization() throws IOException { + public void testSerialization() throws IOException { for (int i = 0; i < 20; i++) { FieldCapabilitiesRequest request = randomRequest(); BytesStreamOutput output = new BytesStreamOutput(); @@ -93,4 +95,11 @@ public void testFieldCapsRequestSerialization() throws IOException { assertEquals(deserialized.hashCode(), request.hashCode()); } } + + public void testValidation() { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() + .indices("index2"); + ActionRequestValidationException exception = request.validate(); + assertNotNull(exception); + } } From 55cad769548438acd76de65b64fc4090be57f8f6 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 14:09:23 -0700 Subject: [PATCH 5/9] FieldCapabilitiesResponseTests now extends AbstractStreamableXContentTestCase. --- .../FieldCapabilitiesResponseTests.java | 130 ++++++++---------- .../fieldcaps/FieldCapabilitiesTests.java | 7 +- 2 files changed, 64 insertions(+), 73 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index adaad7c242792..2860531e865d8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -27,44 +28,79 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.function.Predicate; import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; -public class FieldCapabilitiesResponseTests extends ESTestCase { - private FieldCapabilitiesResponse randomResponse() { - Map > fieldMap = new HashMap<> (); - int numFields = randomInt(10); - for (int i = 0; i < numFields; i++) { - String fieldName = randomAlphaOfLengthBetween(5, 10); - int numIndices = randomIntBetween(1, 5); - Map indexFieldMap = new HashMap<> (); - for (int j = 0; j < numIndices; j++) { - String index = randomAlphaOfLengthBetween(10, 20); - indexFieldMap.put(index, FieldCapabilitiesTests.randomFieldCaps()); +public class FieldCapabilitiesResponseTests extends AbstractStreamableXContentTestCase { + + @Override + protected FieldCapabilitiesResponse doParseInstance(XContentParser parser) throws IOException { + return FieldCapabilitiesResponse.fromXContent(parser); + } + + @Override + protected FieldCapabilitiesResponse createBlankInstance() { + return new FieldCapabilitiesResponse(); + } + + @Override + protected FieldCapabilitiesResponse createTestInstance() { + Map> responses = new HashMap<>(); + + String[] fields = generateRandomStringArray(5, 10, false, true); + assertNotNull(fields); + + for (String field : fields) { + Map typesToCapabilities = new HashMap<>(); + String[] types = generateRandomStringArray(5, 10, false, false); + assertNotNull(types); + + for (String type : types) { + typesToCapabilities.put(type, FieldCapabilitiesTests.randomFieldCaps(field)); } - fieldMap.put(fieldName, indexFieldMap); + responses.put(field, typesToCapabilities); } - return new FieldCapabilitiesResponse(fieldMap); + return new FieldCapabilitiesResponse(responses); } - public void testSerialization() throws IOException { - for (int i = 0; i < 20; i++) { - FieldCapabilitiesResponse response = randomResponse(); - BytesStreamOutput output = new BytesStreamOutput(); - response.writeTo(output); - output.flush(); - StreamInput input = output.bytes().streamInput(); - FieldCapabilitiesResponse deserialized = new FieldCapabilitiesResponse(); - deserialized.readFrom(input); - assertEquals(deserialized, response); - assertEquals(deserialized.hashCode(), response.hashCode()); + @Override + protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse response) { + Map> mutatedResponses = new HashMap<>(response.get()); + + int mutation = response.get().isEmpty() ? 0 : randomIntBetween(0, 2); + + switch (mutation) { + case 0: + String toAdd = randomAlphaOfLength(10); + mutatedResponses.put(toAdd, Collections.singletonMap( + randomAlphaOfLength(10), + FieldCapabilitiesTests.randomFieldCaps(toAdd))); + break; + case 1: + String toRemove = randomFrom(mutatedResponses.keySet()); + mutatedResponses.remove(toRemove); + break; + case 2: + String toReplace = randomFrom(mutatedResponses.keySet()); + mutatedResponses.put(toReplace, Collections.singletonMap( + randomAlphaOfLength(10), + FieldCapabilitiesTests.randomFieldCaps(toReplace))); + break; } + return new FieldCapabilitiesResponse(mutatedResponses); + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.startsWith("fields"); } public void testToXContent() throws IOException { @@ -127,50 +163,4 @@ private static FieldCapabilitiesResponse createSimpleResponse() { responses.put("rating", ratingCapabilities); return new FieldCapabilitiesResponse(responses); } - - public void testFromXContent() throws IOException { - FieldCapabilitiesResponse response = createRandomResponse(); - boolean humanReadable = randomBoolean(); - XContentType xContentType = randomFrom(XContentType.values()); - BytesReference content = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); - - Predicate excludedPaths = path -> path.startsWith("fields"); - BytesReference contentWithRandomFields = insertRandomFields(xContentType, content, excludedPaths, random()); - - FieldCapabilitiesResponse parsedResponse; - try (XContentParser parser = createParser(xContentType.xContent(), contentWithRandomFields)) { - parsedResponse = FieldCapabilitiesResponse.fromXContent(parser); - assertNull(parser.nextToken()); - } - - assertNotSame(response, parsedResponse); - assertEquals(response, parsedResponse); - } - - private static FieldCapabilitiesResponse createRandomResponse() { - Map> responses = new HashMap<>(); - - String[] fields = generateRandomStringArray(5, 10, false, true); - assertNotNull(fields); - - for (String field : fields) { - responses.put(field, new HashMap<>()); - - String[] types = generateRandomStringArray(5, 10, false, false); - assertNotNull(types); - - for (String type : types) { - FieldCapabilities capabilities = new FieldCapabilities(field, type, - randomBoolean(), - randomBoolean(), - generateRandomStringArray(5, 10, true, false), - generateRandomStringArray(3, 10, true, false), - generateRandomStringArray(3, 10, true, false)); - - Map typesToCapabilities = responses.get(field); - typesToCapabilities.put(type, capabilities); - } - } - return new FieldCapabilitiesResponse(responses); - } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index 53c27645bf298..c9753508ef417 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -29,7 +29,8 @@ public class FieldCapabilitiesTests extends AbstractWireSerializingTestCase { @Override protected FieldCapabilities createTestInstance() { - return randomFieldCaps(); + String fieldName = randomAlphaOfLengthBetween(5, 20); + return randomFieldCaps(fieldName); } @Override @@ -82,7 +83,7 @@ public void testBuilder() { } } - static FieldCapabilities randomFieldCaps() { + static FieldCapabilities randomFieldCaps(String fieldName) { String[] indices = null; if (randomBoolean()) { indices = new String[randomIntBetween(1, 5)]; @@ -104,7 +105,7 @@ static FieldCapabilities randomFieldCaps() { nonAggregatableIndices[i] = randomAlphaOfLengthBetween(5, 20); } } - return new FieldCapabilities(randomAlphaOfLengthBetween(5, 20), + return new FieldCapabilities(fieldName, randomAlphaOfLengthBetween(5, 20), randomBoolean(), randomBoolean(), indices, nonSearchableIndices, nonAggregatableIndices); } From 5eaec8300f582839ba24d585a1f5ba8ab5a0663f Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 14:14:15 -0700 Subject: [PATCH 6/9] FieldCapabilitiesTests now extends AbstractSerializingTestCase. --- .../action/fieldcaps/FieldCapabilitiesTests.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java index c9753508ef417..0237ace962a80 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesTests.java @@ -20,17 +20,26 @@ package org.elasticsearch.action.fieldcaps; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; import java.util.Arrays; import static org.hamcrest.Matchers.equalTo; -public class FieldCapabilitiesTests extends AbstractWireSerializingTestCase { +public class FieldCapabilitiesTests extends AbstractSerializingTestCase { + private static final String FIELD_NAME = "field"; + + @Override + protected FieldCapabilities doParseInstance(XContentParser parser) throws IOException { + return FieldCapabilities.fromXContent(FIELD_NAME, parser); + } + @Override protected FieldCapabilities createTestInstance() { - String fieldName = randomAlphaOfLengthBetween(5, 20); - return randomFieldCaps(fieldName); + return randomFieldCaps(FIELD_NAME); } @Override From dd2f34ea55a3f93fdb71bf097dabc49e05d2a161 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 14:45:22 -0700 Subject: [PATCH 7/9] Add information about IndicesOptions to the documentation. --- .../client/documentation/SearchDocumentationIT.java | 4 ++++ docs/java-rest/high-level/search/field-caps.asciidoc | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 12be190dc33f0..4400d05a9f820 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -716,6 +716,10 @@ public void testFieldCaps() throws Exception { .indices("posts", "authors", "contributors"); // end::field-caps-request + // tag::field-caps-request-indicesOptions + request.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::field-caps-request-indicesOptions + // tag::field-caps-execute FieldCapabilitiesResponse response = client.fieldCaps(request); // end::field-caps-execute diff --git a/docs/java-rest/high-level/search/field-caps.asciidoc b/docs/java-rest/high-level/search/field-caps.asciidoc index 75cd85dfbc3fc..fef30f629ca61 100644 --- a/docs/java-rest/high-level/search/field-caps.asciidoc +++ b/docs/java-rest/high-level/search/field-caps.asciidoc @@ -18,6 +18,16 @@ will cause all fields that match the expression to be returned. include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-request] -------------------------------------------------- +[[java-rest-high-field-caps-request-optional]] +===== Optional arguments + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[field-caps-request-indicesOptions] +-------------------------------------------------- +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and +how wildcard expressions are expanded. + [[java-rest-high-field-caps-sync]] ==== Synchronous Execution From aa170dd856914c4618d34792b69ccae2d26adafc Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 15:35:24 -0700 Subject: [PATCH 8/9] Tighten the 'random fields excludes filter' to better test lenient parsing. --- .../action/fieldcaps/FieldCapabilitiesResponseTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 2860531e865d8..c8bd5d5188b67 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.fieldcaps; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -100,7 +101,10 @@ protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse res @Override protected Predicate getRandomFieldsExcludeFilter() { - return field -> field.startsWith("fields"); + // Disallow random fields from being inserted under the 'fields' key, as this + // map only contains field names, and also under 'fields.FIELD_NAME', as these + // maps only contain type names. + return field -> field.matches("fields(\\.\\w+)?"); } public void testToXContent() throws IOException { From ac59e9bcf2bf3c2d47993305c527edf4682c674d Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 24 Apr 2018 15:33:20 -0700 Subject: [PATCH 9/9] Improve a check in RequestTests#testFieldCaps. --- .../elasticsearch/client/RequestTests.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java index f24014a598ff2..0fdeb7555a04a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java @@ -110,11 +110,14 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.StringJoiner; import java.util.function.Consumer; import java.util.function.Function; @@ -1226,28 +1229,32 @@ public void testFieldCaps() { .indices(indices) .fields(fields); - Map expectedIndicesParams = new HashMap<>(); + Map indicesOptionsParams = new HashMap<>(); setRandomIndicesOptions(fieldCapabilitiesRequest::indicesOptions, fieldCapabilitiesRequest::indicesOptions, - expectedIndicesParams); + indicesOptionsParams); Request request = Request.fieldCaps(fieldCapabilitiesRequest); // Verify that the resulting REST request looks as expected. - StringJoiner expectedEndpoint = new StringJoiner("/", "/", ""); + StringJoiner endpoint = new StringJoiner("/", "/", ""); String joinedIndices = String.join(",", indices); if (!joinedIndices.isEmpty()) { - expectedEndpoint.add(joinedIndices); + endpoint.add(joinedIndices); } - expectedEndpoint.add("_field_caps"); + endpoint.add("_field_caps"); - assertEquals(expectedEndpoint.toString(), request.getEndpoint()); + assertEquals(endpoint.toString(), request.getEndpoint()); assertEquals(4, request.getParameters().size()); - // Note that we don't check the field param value explicitly, as field - // names are added to the request in a non-deterministic order. + // Note that we don't check the field param value explicitly, as field names are passed through + // a hash set before being added to the request, and can appear in a non-deterministic order. assertThat(request.getParameters(), hasKey("fields")); - for (Map.Entry param : expectedIndicesParams.entrySet()) { + String[] requestFields = Strings.splitStringByCommaToArray(request.getParameters().get("fields")); + assertEquals(new HashSet<>(Arrays.asList(fields)), + new HashSet<>(Arrays.asList(requestFields))); + + for (Map.Entry param : indicesOptionsParams.entrySet()) { assertThat(request.getParameters(), hasEntry(param.getKey(), param.getValue())); }