From 470d6ba272837b39c11b87729634d8cbaa26fe81 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 3 May 2019 14:22:42 +0200 Subject: [PATCH 1/5] introduce sync API for continuous data frames --- .../DataFrameNamedXContentProvider.java | 41 + .../transforms/DataFrameTransformConfig.java | 44 +- .../dataframe/transforms/SyncConfig.java | 30 + .../dataframe/transforms/TimeSyncConfig.java | 108 ++ ...icsearch.plugins.spi.NamedXContentProvider | 1 + .../DataFrameRequestConvertersTests.java | 7 +- .../client/RestHighLevelClientTests.java | 6 +- .../client/RestHighLevelClientTests.java.orig | 965 ++++++++++++++++++ .../GetDataFrameTransformResponseTests.java | 5 +- ...PreviewDataFrameTransformRequestTests.java | 6 +- .../PutDataFrameTransformRequestTests.java | 6 +- .../DataFrameTransformConfigTests.java | 16 +- .../transforms/TimeSyncConfigTests.java | 49 + .../transforms/hlrc/TimeSyncConfigTests.java | 59 ++ .../DataFrameTransformDocumentationIT.java | 5 +- .../xpack/core/XPackClientPlugin.java | 6 +- .../xpack/core/dataframe/DataFrameField.java | 4 + .../DataFrameNamedXContentProvider.java | 26 + .../transforms/DataFrameTransformConfig.java | 46 +- .../core/dataframe/transforms/SyncConfig.java | 25 + .../dataframe/transforms/TimeSyncConfig.java | 148 +++ ...wDataFrameTransformActionRequestTests.java | 10 +- .../DataFrameTransformConfigTests.java | 26 +- .../transforms/TimeSyncConfigTests.java | 38 + .../integration/DataFrameIntegTestCase.java | 1 + .../DataFrameTransformProgressIT.java | 2 + .../xpack/dataframe/DataFrame.java | 7 + 27 files changed, 1649 insertions(+), 38 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java new file mode 100644 index 0000000000000..940b136c93daa --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/DataFrameNamedXContentProvider.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe; + +import org.elasticsearch.client.dataframe.transforms.SyncConfig; +import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.plugins.spi.NamedXContentProvider; + +import java.util.Arrays; +import java.util.List; + +public class DataFrameNamedXContentProvider implements NamedXContentProvider { + + @Override + public List getNamedXContentParsers() { + return Arrays.asList( + new NamedXContentRegistry.Entry(SyncConfig.class, + new ParseField(TimeSyncConfig.NAME), + TimeSyncConfig::fromXContent)); + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java index 8465ae8342827..da8e5e735d2a0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; import java.util.Objects; @@ -40,12 +41,14 @@ public class DataFrameTransformConfig implements ToXContentObject { public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DEST = new ParseField("dest"); public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField SYNC = new ParseField("sync"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); private final String id; private final SourceConfig source; private final DestConfig dest; + private final SyncConfig syncConfig; private final PivotConfig pivotConfig; private final String description; @@ -55,19 +58,30 @@ public class DataFrameTransformConfig implements ToXContentObject { String id = (String) args[0]; SourceConfig source = (SourceConfig) args[1]; DestConfig dest = (DestConfig) args[2]; - PivotConfig pivotConfig = (PivotConfig) args[3]; - String description = (String)args[4]; - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + SyncConfig syncConfig = (SyncConfig) args[3]; + PivotConfig pivotConfig = (PivotConfig) args[4]; + String description = (String)args[5]; + return new DataFrameTransformConfig(id, source, dest, syncConfig, pivotConfig, description); }); static { PARSER.declareString(constructorArg(), ID); PARSER.declareObject(constructorArg(), (p, c) -> SourceConfig.PARSER.apply(p, null), SOURCE); PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), SYNC); PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); } + private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation); + SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), true); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation); + return syncConfig; + } + + public static DataFrameTransformConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } @@ -84,17 +98,19 @@ public static DataFrameTransformConfig fromXContent(final XContentParser parser) * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new DataFrameTransformConfig(null, source, null, pivotConfig, null); + return new DataFrameTransformConfig(null, source, null, null, pivotConfig, null); } DataFrameTransformConfig(final String id, final SourceConfig source, final DestConfig dest, + final SyncConfig syncConfig, final PivotConfig pivotConfig, final String description) { this.id = id; this.source = source; this.dest = dest; + this.syncConfig = syncConfig; this.pivotConfig = pivotConfig; this.description = description; } @@ -111,6 +127,10 @@ public DestConfig getDestination() { return dest; } + public SyncConfig getSyncConfig() { + return syncConfig; + } + public PivotConfig getPivotConfig() { return pivotConfig; } @@ -132,6 +152,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (dest != null) { builder.field(DEST.getPreferredName(), dest); } + if (syncConfig != null) { + builder.startObject(SYNC.getPreferredName()); + builder.field(syncConfig.getName(), syncConfig); + builder.endObject(); + } if (pivotConfig != null) { builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); } @@ -158,12 +183,13 @@ public boolean equals(Object other) { && Objects.equals(this.source, that.source) && Objects.equals(this.dest, that.dest) && Objects.equals(this.description, that.description) + && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.pivotConfig, that.pivotConfig); } @Override public int hashCode() { - return Objects.hash(id, source, dest, pivotConfig, description); + return Objects.hash(id, source, dest, syncConfig, pivotConfig, description); } @Override @@ -180,6 +206,7 @@ public static class Builder { private String id; private SourceConfig source; private DestConfig dest; + private SyncConfig syncConfig; private PivotConfig pivotConfig; private String description; @@ -198,6 +225,11 @@ public Builder setDest(DestConfig dest) { return this; } + public Builder setSyncConfig(SyncConfig syncConfig) { + this.syncConfig = syncConfig; + return this; + } + public Builder setPivotConfig(PivotConfig pivotConfig) { this.pivotConfig = pivotConfig; return this; @@ -209,7 +241,7 @@ public Builder setDescription(String description) { } public DataFrameTransformConfig build() { - return new DataFrameTransformConfig(id, source, dest, pivotConfig, description); + return new DataFrameTransformConfig(id, source, dest, syncConfig, pivotConfig, description); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java new file mode 100644 index 0000000000000..3ead35d0a491a --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/SyncConfig.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms; + +import org.elasticsearch.common.xcontent.ToXContentObject; + +public interface SyncConfig extends ToXContentObject { + + /** + * Returns the name of the writeable object + */ + String getName(); +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java new file mode 100644 index 0000000000000..797ca3f896138 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfig.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TimeSyncConfig implements SyncConfig { + + public static final String NAME = "time"; + + private static final ParseField FIELD = new ParseField("field"); + private static final ParseField DELAY = new ParseField("delay"); + + private final String field; + private final TimeValue delay; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("time_sync_config", true, + args -> new TimeSyncConfig((String) args[0], args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO)); + + static { + PARSER.declareString(constructorArg(), FIELD); + PARSER.declareField(optionalConstructorArg(), (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DELAY.getPreferredName()), DELAY, + ObjectParser.ValueType.STRING_OR_NULL); + } + + public static TimeSyncConfig fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public TimeSyncConfig(String field, TimeValue delay) { + this.field = field; + this.delay = delay; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD.getPreferredName(), field); + if (delay.duration() > 0) { + builder.field(DELAY.getPreferredName(), delay.getStringRep()); + } + builder.endObject(); + return builder; + } + + public String getField() { + return field; + } + + public TimeValue getDelay() { + return delay; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeSyncConfig that = (TimeSyncConfig) other; + + return Objects.equals(this.field, that.field) + && Objects.equals(this.delay, that.delay); + } + + @Override + public int hashCode() { + return Objects.hash(field, delay); + } + + @Override + public String getName() { + return NAME; + } + +} diff --git a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider index 342c606a540a6..77f1d9700d9a4 100644 --- a/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider +++ b/client/rest-high-level/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider @@ -1,2 +1,3 @@ +org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider org.elasticsearch.client.indexlifecycle.IndexLifecycleNamedXContentProvider org.elasticsearch.client.ml.dataframe.MlDataFrameAnalysisNamedXContentProvider \ No newline at end of file diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java index 8c6b1c6045855..6e48cc507c3e7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java @@ -23,6 +23,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider; import org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest; import org.elasticsearch.client.dataframe.GetDataFrameTransformRequest; import org.elasticsearch.client.dataframe.GetDataFrameTransformStatsRequest; @@ -42,6 +43,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -50,7 +52,10 @@ public class DataFrameRequestConvertersTests extends ESTestCase { @Override protected NamedXContentRegistry xContentRegistry() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); + + return new NamedXContentRegistry(namedXContents); } public void testPutDataFrameTransform() throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index b9dfa4274c28d..5f4bb5ed5e0d7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -46,6 +46,8 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.core.MainRequest; import org.elasticsearch.client.core.MainResponse; +import org.elasticsearch.client.dataframe.transforms.SyncConfig; +import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig; import org.elasticsearch.client.indexlifecycle.AllocateAction; import org.elasticsearch.client.indexlifecycle.DeleteAction; import org.elasticsearch.client.indexlifecycle.ForceMergeAction; @@ -666,7 +668,7 @@ public void testDefaultNamedXContents() { public void testProvidedNamedXContents() { List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); - assertEquals(21, namedXContents.size()); + assertEquals(22, namedXContents.size()); Map, Integer> categories = new HashMap<>(); List names = new ArrayList<>(); for (NamedXContentRegistry.Entry namedXContent : namedXContents) { @@ -702,6 +704,8 @@ public void testProvidedNamedXContents() { assertTrue(names.contains(SetPriorityAction.NAME)); assertEquals(Integer.valueOf(1), categories.get(DataFrameAnalysis.class)); assertTrue(names.contains(OutlierDetection.NAME.getPreferredName())); + assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class)); + assertTrue(names.contains(TimeSyncConfig.NAME)); } public void testApiNamingConventions() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig new file mode 100644 index 0000000000000..f37ef72a547ca --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig @@ -0,0 +1,965 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import com.fasterxml.jackson.core.JsonParseException; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; +import org.apache.http.nio.entity.NByteArrayEntity; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.core.MainRequest; +import org.elasticsearch.client.core.MainResponse; +import org.elasticsearch.client.dataframe.transforms.SyncConfig; +import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig; +import org.elasticsearch.client.indexlifecycle.AllocateAction; +import org.elasticsearch.client.indexlifecycle.DeleteAction; +import org.elasticsearch.client.indexlifecycle.ForceMergeAction; +import org.elasticsearch.client.indexlifecycle.FreezeAction; +import org.elasticsearch.client.indexlifecycle.LifecycleAction; +import org.elasticsearch.client.indexlifecycle.ReadOnlyAction; +import org.elasticsearch.client.indexlifecycle.RolloverAction; +import org.elasticsearch.client.indexlifecycle.SetPriorityAction; +import org.elasticsearch.client.indexlifecycle.ShrinkAction; +import org.elasticsearch.client.indexlifecycle.UnfollowAction; +import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis; +import org.elasticsearch.client.ml.dataframe.OutlierDetection; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.cbor.CborXContent; +import org.elasticsearch.common.xcontent.smile.SmileXContent; +import org.elasticsearch.index.rankeval.DiscountedCumulativeGain; +import org.elasticsearch.index.rankeval.EvaluationMetric; +import org.elasticsearch.index.rankeval.ExpectedReciprocalRank; +import org.elasticsearch.index.rankeval.MeanReciprocalRank; +import org.elasticsearch.index.rankeval.MetricDetail; +import org.elasticsearch.index.rankeval.PrecisionAtK; +import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.InternalAggregationTestCase; +import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; +import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; +import org.hamcrest.Matchers; +import org.junit.Before; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.hamcrest.CoreMatchers.endsWith; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RestHighLevelClientTests extends ESTestCase { + + private static final String SUBMIT_TASK_PREFIX = "submit_"; + private static final String SUBMIT_TASK_SUFFIX = "_task"; + private static final ProtocolVersion HTTP_PROTOCOL = new ProtocolVersion("http", 1, 1); + private static final RequestLine REQUEST_LINE = new BasicRequestLine(HttpGet.METHOD_NAME, "/", HTTP_PROTOCOL); + + /** + * These APIs do not use a Request object (because they don't have a body, or any request parameters). + * The method naming/parameter assertions use this {@code Set} to determine which rules to apply. + * (This is also used for async variants of these APIs when they exist) + */ + private static final Set APIS_WITHOUT_REQUEST_OBJECT = Sets.newHashSet( + // core + "ping", "info", + // security + "security.get_ssl_certificates", "security.authenticate", "security.get_user_privileges", + // license + "license.get_trial_status", "license.get_basic_status" + + ); + + private RestClient restClient; + private RestHighLevelClient restHighLevelClient; + + @Before + public void initClient() { + restClient = mock(RestClient.class); + restHighLevelClient = new RestHighLevelClient(restClient, RestClient::close, Collections.emptyList()); + } + + public void testCloseIsIdempotent() throws IOException { + restHighLevelClient.close(); + verify(restClient, times(1)).close(); + restHighLevelClient.close(); + verify(restClient, times(2)).close(); + restHighLevelClient.close(); + verify(restClient, times(3)).close(); + } + + public void testPingSuccessful() throws IOException { + Response response = mock(Response.class); + when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.OK)); + when(restClient.performRequest(any(Request.class))).thenReturn(response); + assertTrue(restHighLevelClient.ping(RequestOptions.DEFAULT)); + } + + public void testPing404NotFound() throws IOException { + Response response = mock(Response.class); + when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.NOT_FOUND)); + when(restClient.performRequest(any(Request.class))).thenReturn(response); + assertFalse(restHighLevelClient.ping(RequestOptions.DEFAULT)); + } + + public void testPingSocketTimeout() throws IOException { + when(restClient.performRequest(any(Request.class))).thenThrow(new SocketTimeoutException()); + expectThrows(SocketTimeoutException.class, () -> restHighLevelClient.ping(RequestOptions.DEFAULT)); + } + + public void testInfo() throws IOException { + MainResponse testInfo = new MainResponse("nodeName", new MainResponse.Version("number", "buildFlavor", "buildType", "buildHash", + "buildDate", true, "luceneVersion", "minimumWireCompatibilityVersion", "minimumIndexCompatibilityVersion"), + "clusterName", "clusterUuid", "You Know, for Search"); + mockResponse((ToXContentFragment) (builder, params) -> { + // taken from the server side MainResponse + builder.field("name", testInfo.getNodeName()); + builder.field("cluster_name", testInfo.getClusterName()); + builder.field("cluster_uuid", testInfo.getClusterUuid()); + builder.startObject("version") + .field("number", testInfo.getVersion().getNumber()) + .field("build_flavor", testInfo.getVersion().getBuildFlavor()) + .field("build_type", testInfo.getVersion().getBuildType()) + .field("build_hash", testInfo.getVersion().getBuildHash()) + .field("build_date", testInfo.getVersion().getBuildDate()) + .field("build_snapshot", testInfo.getVersion().isSnapshot()) + .field("lucene_version", testInfo.getVersion().getLuceneVersion()) + .field("minimum_wire_compatibility_version", testInfo.getVersion().getMinimumWireCompatibilityVersion()) + .field("minimum_index_compatibility_version", testInfo.getVersion().getMinimumIndexCompatibilityVersion()) + .endObject(); + builder.field("tagline", testInfo.getTagline()); + return builder; + }); + MainResponse receivedInfo = restHighLevelClient.info(RequestOptions.DEFAULT); + assertEquals(testInfo, receivedInfo); + } + + public void testSearchScroll() throws IOException { + SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY, + null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY); + mockResponse(mockSearchResponse); + SearchResponse searchResponse = restHighLevelClient.scroll( + new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), RequestOptions.DEFAULT); + assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId()); + assertEquals(0, searchResponse.getHits().getTotalHits().value); + assertEquals(5, searchResponse.getTotalShards()); + assertEquals(5, searchResponse.getSuccessfulShards()); + assertEquals(100, searchResponse.getTook().getMillis()); + } + + public void testClearScroll() throws IOException { + ClearScrollResponse mockClearScrollResponse = new ClearScrollResponse(randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE)); + mockResponse(mockClearScrollResponse); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(5, 10)); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + assertEquals(mockClearScrollResponse.isSucceeded(), clearScrollResponse.isSucceeded()); + assertEquals(mockClearScrollResponse.getNumFreed(), clearScrollResponse.getNumFreed()); + } + + private void mockResponse(ToXContent toXContent) throws IOException { + Response response = mock(Response.class); + ContentType contentType = ContentType.parse(RequestConverters.REQUEST_BODY_CONTENT_TYPE.mediaType()); + String requestBody = toXContent(toXContent, RequestConverters.REQUEST_BODY_CONTENT_TYPE, false).utf8ToString(); + when(response.getEntity()).thenReturn(new NStringEntity(requestBody, contentType)); + when(restClient.performRequest(any(Request.class))).thenReturn(response); + } + + public void testRequestValidation() { + ActionRequestValidationException validationException = new ActionRequestValidationException(); + validationException.addValidationError("validation error"); + ActionRequest request = new ActionRequest() { + @Override + public ActionRequestValidationException validate() { + return validationException; + } + }; + + { + ActionRequestValidationException actualException = expectThrows(ActionRequestValidationException.class, + () -> restHighLevelClient.performRequest(request, null, RequestOptions.DEFAULT, null, null)); + assertSame(validationException, actualException); + } + { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + restHighLevelClient.performRequestAsync(request, null, RequestOptions.DEFAULT, null, trackingActionListener, null); + assertSame(validationException, trackingActionListener.exception.get()); + } + } + + public void testParseEntity() throws IOException { + { + IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(null, null)); + assertEquals("Response body expected but not returned", ise.getMessage()); + } + { + IllegalStateException ise = expectThrows(IllegalStateException.class, + () -> restHighLevelClient.parseEntity(new NStringEntity("", (ContentType) null), null)); + assertEquals("Elasticsearch didn't return the [Content-Type] header, unable to parse response body", ise.getMessage()); + } + { + NStringEntity entity = new NStringEntity("", ContentType.APPLICATION_SVG_XML); + IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(entity, null)); + assertEquals("Unsupported Content-Type: " + entity.getContentType().getValue(), ise.getMessage()); + } + { + CheckedFunction entityParser = parser -> { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertTrue(parser.nextToken().isValue()); + String value = parser.text(); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + return value; + }; + HttpEntity jsonEntity = new NStringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON); + assertEquals("value", restHighLevelClient.parseEntity(jsonEntity, entityParser)); + HttpEntity yamlEntity = new NStringEntity("---\nfield: value\n", ContentType.create("application/yaml")); + assertEquals("value", restHighLevelClient.parseEntity(yamlEntity, entityParser)); + HttpEntity smileEntity = createBinaryEntity(SmileXContent.contentBuilder(), ContentType.create("application/smile")); + assertEquals("value", restHighLevelClient.parseEntity(smileEntity, entityParser)); + HttpEntity cborEntity = createBinaryEntity(CborXContent.contentBuilder(), ContentType.create("application/cbor")); + assertEquals("value", restHighLevelClient.parseEntity(cborEntity, entityParser)); + } + } + + private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, ContentType contentType) throws IOException { + try (XContentBuilder builder = xContentBuilder) { + builder.startObject(); + builder.field("field", "value"); + builder.endObject(); + return new NByteArrayEntity(BytesReference.bytes(builder).toBytesRef().bytes, contentType); + } + } + + public void testConvertExistsResponse() { + RestStatus restStatus = randomBoolean() ? RestStatus.OK : randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + boolean result = RestHighLevelClient.convertExistsResponse(response); + assertEquals(restStatus == RestStatus.OK, result); + } + + public void testParseResponseException() throws IOException { + { + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); + assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + } + { + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", + ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); + assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getSuppressed()[0]); + } + { + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IOException.class)); + } + { + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); + } + } + + public void testPerformRequestOnSuccess() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + when(restClient.performRequest(any(Request.class))).thenReturn(mockResponse); + { + Integer result = restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.emptySet()); + assertEquals(restStatus.getStatus(), result.intValue()); + } + { + IOException ioe = expectThrows(IOException.class, () -> restHighLevelClient.performRequest(mainRequest, + requestConverter, RequestOptions.DEFAULT, response -> {throw new IllegalStateException();}, Collections.emptySet())); + assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " + + "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage()); + } + } + + public void testPerformRequestOnResponseExceptionWithoutEntity() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); + assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + } + + public void testPerformRequestOnResponseExceptionWithEntity() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", + ContentType.APPLICATION_JSON)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); + assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getSuppressed()[0]); + } + + public void testPerformRequestOnResponseExceptionWithBrokenEntity() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class)); + } + + public void testPerformRequestOnResponseExceptionWithBrokenEntity2() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); + } + + public void testPerformRequestOnResponseExceptionWithIgnores() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + //although we got an exception, we turn it into a successful response because the status code was provided among ignores + assertEquals(Integer.valueOf(404), restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> response.getStatusLine().getStatusCode(), Collections.singleton(404))); + } + + public void testPerformRequestOnResponseExceptionWithIgnoresErrorNoBody() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> {throw new IllegalStateException();}, Collections.singleton(404))); + assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); + } + + public void testPerformRequestOnResponseExceptionWithIgnoresErrorValidBody() throws IOException { + MainRequest mainRequest = new MainRequest(); + CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":404}", + ContentType.APPLICATION_JSON)); + Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(mockResponse); + when(restClient.performRequest(any(Request.class))).thenThrow(responseException); + ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, + response -> {throw new IllegalStateException();}, Collections.singleton(404))); + assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getSuppressed()[0]); + assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); + } + + public void testWrapResponseListenerOnSuccess() { + { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse)); + assertNull(trackingActionListener.exception.get()); + assertEquals(restStatus.getStatus(), trackingActionListener.statusCode.get()); + } + { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> {throw new IllegalStateException();}, trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse)); + assertThat(trackingActionListener.exception.get(), instanceOf(IOException.class)); + IOException ioe = (IOException) trackingActionListener.exception.get(); + assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " + + "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage()); + assertThat(ioe.getCause(), instanceOf(IllegalStateException.class)); + } + } + + public void testWrapResponseListenerOnException() { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + IllegalStateException exception = new IllegalStateException(); + responseListener.onFailure(exception); + assertSame(exception, trackingActionListener.exception.get()); + } + + public void testWrapResponseListenerOnResponseExceptionWithoutEntity() throws IOException { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException) trackingActionListener.exception.get(); + assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + } + + public void testWrapResponseListenerOnResponseExceptionWithEntity() throws IOException { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", + ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); + assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getSuppressed()[0]); + } + + public void testWrapResponseListenerOnResponseExceptionWithBrokenEntity() throws IOException { + { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class)); + } + { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); + RestStatus restStatus = randomFrom(RestStatus.values()); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); + httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); + assertEquals("Unable to parse response body", elasticsearchException.getMessage()); + assertEquals(restStatus, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); + } + } + + public void testWrapResponseListenerOnResponseExceptionWithIgnores() throws IOException { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.singleton(404)); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + //although we got an exception, we turn it into a successful response because the status code was provided among ignores + assertNull(trackingActionListener.exception.get()); + assertEquals(404, trackingActionListener.statusCode.get()); + } + + public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorNoBody() throws IOException { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying + //to parse a 404 response which contains an error rather than a valid document not found response. + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404)); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); + assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getCause()); + assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); + } + + public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorValidBody() throws IOException { + TrackingActionListener trackingActionListener = new TrackingActionListener(); + //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying + //to parse a 404 response which contains an error rather than a valid document not found response. + ResponseListener responseListener = restHighLevelClient.wrapResponseListener( + response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404)); + HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); + httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":404}", + ContentType.APPLICATION_JSON)); + Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); + ResponseException responseException = new ResponseException(response); + responseListener.onFailure(responseException); + assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); + ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); + assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); + assertSame(responseException, elasticsearchException.getSuppressed()[0]); + assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); + } + + public void testDefaultNamedXContents() { + List namedXContents = RestHighLevelClient.getDefaultNamedXContents(); + int expectedInternalAggregations = InternalAggregationTestCase.getDefaultNamedXContents().size(); + int expectedSuggestions = 3; + assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size()); + Map, Integer> categories = new HashMap<>(); + for (NamedXContentRegistry.Entry namedXContent : namedXContents) { + Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1); + if (counter != null) { + categories.put(namedXContent.categoryClass, counter + 1); + } + } + assertEquals(2, categories.size()); + assertEquals(expectedInternalAggregations, categories.get(Aggregation.class).intValue()); + assertEquals(expectedSuggestions, categories.get(Suggest.Suggestion.class).intValue()); + } + + public void testProvidedNamedXContents() { + List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); + assertEquals(21, namedXContents.size()); + Map, Integer> categories = new HashMap<>(); + List names = new ArrayList<>(); + for (NamedXContentRegistry.Entry namedXContent : namedXContents) { + names.add(namedXContent.name.getPreferredName()); + Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1); + if (counter != null) { + categories.put(namedXContent.categoryClass, counter + 1); + } + } + assertEquals("Had: " + categories, 5, categories.size()); + assertEquals(Integer.valueOf(3), categories.get(Aggregation.class)); + assertTrue(names.contains(ChildrenAggregationBuilder.NAME)); + assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME)); + assertEquals(Integer.valueOf(4), categories.get(EvaluationMetric.class)); + assertTrue(names.contains(PrecisionAtK.NAME)); + assertTrue(names.contains(DiscountedCumulativeGain.NAME)); + assertTrue(names.contains(MeanReciprocalRank.NAME)); + assertTrue(names.contains(ExpectedReciprocalRank.NAME)); + assertEquals(Integer.valueOf(4), categories.get(MetricDetail.class)); + assertTrue(names.contains(PrecisionAtK.NAME)); + assertTrue(names.contains(MeanReciprocalRank.NAME)); + assertTrue(names.contains(DiscountedCumulativeGain.NAME)); + assertTrue(names.contains(ExpectedReciprocalRank.NAME)); + assertEquals(Integer.valueOf(9), categories.get(LifecycleAction.class)); + assertTrue(names.contains(UnfollowAction.NAME)); + assertTrue(names.contains(AllocateAction.NAME)); + assertTrue(names.contains(DeleteAction.NAME)); + assertTrue(names.contains(ForceMergeAction.NAME)); + assertTrue(names.contains(ReadOnlyAction.NAME)); + assertTrue(names.contains(RolloverAction.NAME)); + assertTrue(names.contains(ShrinkAction.NAME)); + assertTrue(names.contains(FreezeAction.NAME)); + assertTrue(names.contains(SetPriorityAction.NAME)); +<<<<<<< HEAD + assertEquals(Integer.valueOf(1), categories.get(DataFrameAnalysis.class)); + assertTrue(names.contains(OutlierDetection.NAME.getPreferredName())); +======= + assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class)); + assertTrue(names.contains(TimeSyncConfig.NAME)); +>>>>>>> d94764e95aa... introduce sync API + } + + public void testApiNamingConventions() throws Exception { + //this list should be empty once the high-level client is feature complete + String[] notYetSupportedApi = new String[]{ + "cluster.remote_info", + "create", + "get_source", + "indices.delete_alias", + "indices.exists_type", + "indices.get_upgrade", + "indices.put_alias", + "render_search_template", + "scripts_painless_execute" + }; + //These API are not required for high-level client feature completeness + String[] notRequiredApi = new String[] { + "cluster.allocation_explain", + "cluster.pending_tasks", + "cluster.reroute", + "cluster.state", + "cluster.stats", + "indices.shard_stores", + "indices.upgrade", + "indices.recovery", + "indices.segments", + "indices.stats", + "ingest.processor_grok", + "nodes.info", + "nodes.stats", + "nodes.hot_threads", + "nodes.usage", + "nodes.reload_secure_settings", + "scripts_painless_context", + "search_shards", + }; + List booleanReturnMethods = Arrays.asList( + "security.enable_user", + "security.disable_user", + "security.change_password"); + Set deprecatedMethods = new HashSet<>(); + deprecatedMethods.add("indices.force_merge"); + deprecatedMethods.add("multi_get"); + deprecatedMethods.add("multi_search"); + deprecatedMethods.add("search_scroll"); + + ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load("/rest-api-spec/api"); + Set apiSpec = restSpec.getApis().stream().map(ClientYamlSuiteRestApi::getName).collect(Collectors.toSet()); + Set apiUnsupported = new HashSet<>(apiSpec); + Set apiNotFound = new HashSet<>(); + + Set topLevelMethodsExclusions = new HashSet<>(); + topLevelMethodsExclusions.add("getLowLevelClient"); + topLevelMethodsExclusions.add("close"); + + Map> methods = Arrays.stream(RestHighLevelClient.class.getMethods()) + .filter(method -> method.getDeclaringClass().equals(RestHighLevelClient.class) + && topLevelMethodsExclusions.contains(method.getName()) == false) + .map(method -> Tuple.tuple(toSnakeCase(method.getName()), method)) + .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client") + ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple)) + .filter(tuple -> tuple.v2().getAnnotation(Deprecated.class) == null) + .collect(Collectors.groupingBy(Tuple::v1, + Collectors.mapping(Tuple::v2, Collectors.toSet()))); + + // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate + // impl will replace the existing getTemplate method. + // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` + // looking like it doesn't have a valid implementatation when it does. + apiUnsupported.remove("indices.get_template"); + + + + for (Map.Entry> entry : methods.entrySet()) { + String apiName = entry.getKey(); + + for (Method method : entry.getValue()) { + assertTrue("method [" + apiName + "] is not final", + Modifier.isFinal(method.getClass().getModifiers()) || Modifier.isFinal(method.getModifiers())); + assertTrue("method [" + method + "] should be public", Modifier.isPublic(method.getModifiers())); + + //we convert all the method names to snake case, hence we need to look for the '_async' suffix rather than 'Async' + if (apiName.endsWith("_async")) { + assertAsyncMethod(methods, method, apiName); + } else if (isSubmitTaskMethod(apiName)) { + assertSubmitTaskMethod(methods, method, apiName, restSpec); + } else { + assertSyncMethod(method, apiName, booleanReturnMethods); + apiUnsupported.remove(apiName); + if (apiSpec.contains(apiName) == false) { + if (deprecatedMethods.contains(apiName)) { + assertTrue("method [" + method.getName() + "], api [" + apiName + "] should be deprecated", + method.isAnnotationPresent(Deprecated.class)); + } else { + //TODO xpack api are currently ignored, we need to load xpack yaml spec too + if (apiName.startsWith("xpack.") == false && + apiName.startsWith("license.") == false && + apiName.startsWith("machine_learning.") == false && + apiName.startsWith("rollup.") == false && + apiName.startsWith("watcher.") == false && + apiName.startsWith("graph.") == false && + apiName.startsWith("migration.") == false && + apiName.startsWith("security.") == false && + apiName.startsWith("index_lifecycle.") == false && + apiName.startsWith("ccr.") == false && + apiName.startsWith("data_frame") == false && + apiName.endsWith("freeze") == false && + // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we + // can get rid of 7.0's deprecated "getTemplate" + apiName.equals("indices.get_index_template") == false) { + apiNotFound.add(apiName); + } + } + } + } + } + } + assertThat("Some client method doesn't match a corresponding API defined in the REST spec: " + apiNotFound, + apiNotFound.size(), equalTo(0)); + + //we decided not to support cat API in the high-level REST client, they are supposed to be used from a low-level client + apiUnsupported.removeIf(api -> api.startsWith("cat.")); + Stream.concat(Arrays.stream(notYetSupportedApi), Arrays.stream(notRequiredApi)).forEach( + api -> assertTrue(api + " API is either not defined in the spec or already supported by the high-level client", + apiUnsupported.remove(api))); + assertThat("Some API are not supported but they should be: " + apiUnsupported, apiUnsupported.size(), equalTo(0)); + } + + private static void assertSyncMethod(Method method, String apiName, List booleanReturnMethods) { + //A few methods return a boolean rather than a response object + if (apiName.equals("ping") || apiName.contains("exist") || booleanReturnMethods.contains(apiName)) { + assertThat("the return type for method [" + method + "] is incorrect", + method.getReturnType().getSimpleName(), equalTo("boolean")); + } else { + // It's acceptable for 404s to be represented as empty Optionals + if (!method.getReturnType().isAssignableFrom(Optional.class)) { + assertThat("the return type for method [" + method + "] is incorrect", + method.getReturnType().getSimpleName(), endsWith("Response")); + } + } + + assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length); + //a few methods don't accept a request object as argument + if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName)) { + assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length); + assertThat("the parameter to method [" + method + "] is the wrong type", + method.getParameterTypes()[0], equalTo(RequestOptions.class)); + } else { + assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterTypes().length); + // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation + if (method.getParameterTypes()[0].equals(RequestOptions.class)) { + assertThat("the first parameter to method [" + method + "] is the wrong type", + method.getParameterTypes()[0], equalTo(RequestOptions.class)); + assertThat("the second parameter to method [" + method + "] is the wrong type", + method.getParameterTypes()[1].getSimpleName(), endsWith("Request")); + } else { + assertThat("the first parameter to method [" + method + "] is the wrong type", + method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); + assertThat("the second parameter to method [" + method + "] is the wrong type", + method.getParameterTypes()[1], equalTo(RequestOptions.class)); + } + } + } + + private static void assertAsyncMethod(Map> methods, Method method, String apiName) { + assertTrue("async method [" + method.getName() + "] doesn't have corresponding sync method", + methods.containsKey(apiName.substring(0, apiName.length() - 6))); + assertThat("async method [" + method + "] should return void", method.getReturnType(), equalTo(Void.TYPE)); + assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length); + if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName.replaceAll("_async$", ""))) { + assertEquals(2, method.getParameterTypes().length); + assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class)); + assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class)); + } else { + assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterTypes().length); + // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation + if (method.getParameterTypes()[0].equals(RequestOptions.class)) { + assertThat("the first parameter to async method [" + method + "] should be a request type", + method.getParameterTypes()[0], equalTo(RequestOptions.class)); + assertThat("the second parameter to async method [" + method + "] is the wrong type", + method.getParameterTypes()[1].getSimpleName(), endsWith("Request")); + } else { + assertThat("the first parameter to async method [" + method + "] should be a request type", + method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); + assertThat("the second parameter to async method [" + method + "] is the wrong type", + method.getParameterTypes()[1], equalTo(RequestOptions.class)); + } + assertThat("the third parameter to async method [" + method + "] is the wrong type", + method.getParameterTypes()[2], equalTo(ActionListener.class)); + } + } + + private static void assertSubmitTaskMethod(Map> methods, Method method, String apiName, + ClientYamlSuiteRestSpec restSpec) { + String methodName = extractMethodName(apiName); + assertTrue("submit task method [" + method.getName() + "] doesn't have corresponding sync method", + methods.containsKey(methodName)); + assertEquals("submit task method [" + method + "] has the wrong number of arguments", 2, method.getParameterTypes().length); + assertThat("the first parameter to submit task method [" + method + "] is the wrong type", + method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); + assertThat("the second parameter to submit task method [" + method + "] is the wrong type", + method.getParameterTypes()[1], equalTo(RequestOptions.class)); + + assertThat("submit task method [" + method + "] must have wait_for_completion parameter in rest spec", + restSpec.getApi(methodName).getParams(), Matchers.hasKey("wait_for_completion")); + } + + private static String extractMethodName(String apiName) { + return apiName.substring(SUBMIT_TASK_PREFIX.length(), apiName.length() - SUBMIT_TASK_SUFFIX.length()); + } + + private static boolean isSubmitTaskMethod(String apiName) { + return apiName.startsWith(SUBMIT_TASK_PREFIX) && apiName.endsWith(SUBMIT_TASK_SUFFIX); + } + + private static Stream> getSubClientMethods(String namespace, Class clientClass) { + return Arrays.stream(clientClass.getMethods()).filter(method -> method.getDeclaringClass().equals(clientClass)) + .map(method -> Tuple.tuple(namespace + "." + toSnakeCase(method.getName()), method)) + .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client") + ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple)); + } + + private static String toSnakeCase(String camelCase) { + StringBuilder snakeCaseString = new StringBuilder(); + for (Character aChar : camelCase.toCharArray()) { + if (Character.isUpperCase(aChar)) { + snakeCaseString.append('_'); + snakeCaseString.append(Character.toLowerCase(aChar)); + } else { + snakeCaseString.append(aChar); + } + } + return snakeCaseString.toString(); + } + + private static class TrackingActionListener implements ActionListener { + private final AtomicInteger statusCode = new AtomicInteger(-1); + private final AtomicReference exception = new AtomicReference<>(); + + @Override + public void onResponse(Integer statusCode) { + assertTrue(this.statusCode.compareAndSet(-1, statusCode)); + } + + @Override + public void onFailure(Exception e) { + assertTrue(exception.compareAndSet(null, e)); + } + } + + private static StatusLine newStatusLine(RestStatus restStatus) { + return new BasicStatusLine(HTTP_PROTOCOL, restStatus.getStatus(), restStatus.name()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java index f7386e936301b..27b2cc9b99bf8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/GetDataFrameTransformResponseTests.java @@ -79,6 +79,9 @@ private static void toXContent(GetDataFrameTransformResponse response, XContentB @Override protected NamedXContentRegistry xContentRegistry() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); + + return new NamedXContentRegistry(namedXContents); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java index c91e1cbb1dd91..45d5d879d47f9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.elasticsearch.client.dataframe.transforms.SourceConfigTests.randomSourceConfig; @@ -55,7 +56,10 @@ protected boolean supportsUnknownFields() { @Override protected NamedXContentRegistry xContentRegistry() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); + + return new NamedXContentRegistry(namedXContents); } public void testValidate() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java index 28fd92dcf913f..7c7cd3fa151fe 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.hamcrest.Matchers.containsString; @@ -71,6 +72,9 @@ protected boolean supportsUnknownFields() { @Override protected NamedXContentRegistry xContentRegistry() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); + + return new NamedXContentRegistry(namedXContents); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java index 1b5228d96229f..803ad39b9fb63 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.dataframe.transforms; +import org.elasticsearch.client.dataframe.DataFrameNamedXContentProvider; import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfigTests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -28,6 +29,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.function.Predicate; import static org.elasticsearch.client.dataframe.transforms.DestConfigTests.randomDestConfig; @@ -36,8 +38,13 @@ public class DataFrameTransformConfigTests extends AbstractXContentTestCase { public static DataFrameTransformConfig randomDataFrameTransformConfig() { - return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), - randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), randomDestConfig(), + randomBoolean() ? randomSyncConfig() : null, PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + } + + public static SyncConfig randomSyncConfig() { + return TimeSyncConfigTests.randomTimeSyncConfig(); } @Override @@ -64,6 +71,9 @@ protected Predicate getRandomFieldsExcludeFilter() { @Override protected NamedXContentRegistry xContentRegistry() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); - return new NamedXContentRegistry(searchModule.getNamedXContents()); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); + + return new NamedXContentRegistry(namedXContents); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java new file mode 100644 index 0000000000000..dd2a17eb0260d --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/TimeSyncConfigTests.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class TimeSyncConfigTests extends AbstractXContentTestCase { + + public static TimeSyncConfig randomTimeSyncConfig() { + return new TimeSyncConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + } + + @Override + protected TimeSyncConfig createTestInstance() { + return randomTimeSyncConfig(); + } + + @Override + protected TimeSyncConfig doParseInstance(XContentParser parser) throws IOException { + return TimeSyncConfig.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java new file mode 100644 index 0000000000000..0c6a0350882a4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/hlrc/TimeSyncConfigTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.dataframe.transforms.hlrc; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +public class TimeSyncConfigTests + extends AbstractResponseTestCase { + + public static org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig randomTimeSyncConfig() { + return new org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig(randomAlphaOfLengthBetween(1, 10), + new TimeValue(randomNonNegativeLong())); + } + + public static void assertHlrcEquals(org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig serverTestInstance, + TimeSyncConfig clientInstance) { + assertEquals(serverTestInstance.getField(), clientInstance.getField()); + assertEquals(serverTestInstance.getDelay(), clientInstance.getDelay()); + } + + @Override + protected org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig createServerTestInstance() { + return randomTimeSyncConfig(); + } + + @Override + protected TimeSyncConfig doParseToClientInstance(XContentParser parser) throws IOException { + return TimeSyncConfig.fromXContent(parser); + } + + @Override + protected void assertInstances(org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig serverTestInstance, + TimeSyncConfig clientInstance) { + assertHlrcEquals(serverTestInstance, clientInstance); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java index 60dd2cb32eaab..a41540af3fb4a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java @@ -422,6 +422,7 @@ public void testPreview() throws IOException, InterruptedException { .setQueryConfig(queryConfig) .build(), // <1> pivotConfig); // <2> + PreviewDataFrameTransformRequest request = new PreviewDataFrameTransformRequest(transformConfig); // <3> // end::preview-data-frame-transform-request @@ -469,7 +470,6 @@ public void testGetStats() throws IOException, InterruptedException { RestHighLevelClient client = highLevelClient(); - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); @@ -554,8 +554,7 @@ public void onFailure(Exception e) { public void testGetDataFrameTransform() throws IOException, InterruptedException { createIndex("source-data"); - - QueryConfig queryConfig = new QueryConfig(new MatchAllQueryBuilder()); + GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 74c2b3d6af407..07fac1baa8b58 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -54,6 +54,8 @@ import org.elasticsearch.xpack.core.dataframe.action.StopDataFrameTransformAction; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransform; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformState; +import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage; import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; @@ -511,7 +513,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_FRAME, DataFrameFeatureSetUsage::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, DataFrameField.TASK_NAME, DataFrameTransform::new), new NamedWriteableRegistry.Entry(Task.Status.class, DataFrameField.TASK_NAME, DataFrameTransformState::new), - new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new) + new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new), + new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) + ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java index 71bf14cdeb4a5..d6dd5a30f5d65 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java @@ -27,6 +27,10 @@ public final class DataFrameField { public static final ParseField SOURCE = new ParseField("source"); public static final ParseField DESTINATION = new ParseField("dest"); public static final ParseField FORCE = new ParseField("force"); + public static final ParseField FIELD = new ParseField("field"); + public static final ParseField SYNC = new ParseField("sync"); + public static final ParseField TIME_BASED_SYNC = new ParseField("time"); + public static final ParseField DELAY = new ParseField("delay"); /** * Fields for checkpointing diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java new file mode 100644 index 0000000000000..9eacfc5ff1eae --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameNamedXContentProvider.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe; + +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.plugins.spi.NamedXContentProvider; +import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; + +import java.util.Arrays; +import java.util.List; + +public class DataFrameNamedXContentProvider implements NamedXContentProvider { + + @Override + public List getNamedXContentParsers() { + return Arrays.asList( + new NamedXContentRegistry.Entry(SyncConfig.class, + DataFrameField.TIME_BASED_SYNC, + TimeSyncConfig::parse)); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java index ee35fe3d21ec7..2750daea6b3fd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfig; @@ -49,6 +50,7 @@ public class DataFrameTransformConfig extends AbstractDiffable create SourceConfig source = (SourceConfig) args[1]; DestConfig dest = (DestConfig) args[2]; - // ignored, only for internal storage: String docType = (String) args[3]; + SyncConfig syncConfig = (SyncConfig) args[3]; + // ignored, only for internal storage: String docType = (String) args[4]; // on strict parsing do not allow injection of headers - if (lenient == false && args[4] != null) { + if (lenient == false && args[5] != null) { throw new IllegalArgumentException("Found [headers], not allowed for strict parsing"); } @SuppressWarnings("unchecked") - Map headers = (Map) args[4]; + Map headers = (Map) args[5]; - PivotConfig pivotConfig = (PivotConfig) args[5]; - String description = (String)args[6]; - return new DataFrameTransformConfig(id, source, dest, headers, pivotConfig, description); + PivotConfig pivotConfig = (PivotConfig) args[6]; + String description = (String)args[7]; + return new DataFrameTransformConfig(id, source, dest, syncConfig, headers, pivotConfig, description); }); parser.declareString(optionalConstructorArg(), DataFrameField.ID); parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), DataFrameField.SOURCE); parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), DataFrameField.DESTINATION); + parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), DataFrameField.SYNC); + parser.declareString(optionalConstructorArg(), DataFrameField.INDEX_DOC_TYPE); + parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareString(optionalConstructorArg(), DESCRIPTION); @@ -99,6 +105,14 @@ private static ConstructingObjectParser create return parser; } + private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation); + SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation); + return syncConfig; + } + public static String documentId(String transformId) { return NAME + "-" + transformId; } @@ -106,12 +120,14 @@ public static String documentId(String transformId) { public DataFrameTransformConfig(final String id, final SourceConfig source, final DestConfig dest, + final SyncConfig syncConfig, final Map headers, final PivotConfig pivotConfig, final String description) { this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName()); this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName()); this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName()); + this.syncConfig = syncConfig; this.setHeaders(headers == null ? Collections.emptyMap() : headers); this.pivotConfig = pivotConfig; this.description = description; @@ -129,6 +145,7 @@ public DataFrameTransformConfig(final StreamInput in) throws IOException { id = in.readString(); source = new SourceConfig(in); dest = new DestConfig(in); + syncConfig = in.readOptionalNamedWriteable(SyncConfig.class); setHeaders(in.readMap(StreamInput::readString, StreamInput::readString)); pivotConfig = in.readOptionalWriteable(PivotConfig::new); description = in.readOptionalString(); @@ -146,6 +163,10 @@ public DestConfig getDestination() { return dest; } + public SyncConfig getSyncConfig() { + return syncConfig; + } + public Map getHeaders() { return headers; } @@ -168,6 +189,10 @@ public boolean isValid() { return false; } + if (syncConfig != null && syncConfig.isValid() == false) { + return false; + } + return source.isValid() && dest.isValid(); } @@ -176,6 +201,7 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(id); source.writeTo(out); dest.writeTo(out); + out.writeOptionalNamedWriteable(syncConfig); out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); out.writeOptionalWriteable(pivotConfig); out.writeOptionalString(description); @@ -187,6 +213,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(DataFrameField.ID.getPreferredName(), id); builder.field(DataFrameField.SOURCE.getPreferredName(), source); builder.field(DataFrameField.DESTINATION.getPreferredName(), dest); + if (syncConfig != null) { + builder.startObject(DataFrameField.SYNC.getPreferredName()); + builder.field(syncConfig.getWriteableName(), syncConfig); + builder.endObject(); + } if (pivotConfig != null) { builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); } @@ -218,6 +249,7 @@ public boolean equals(Object other) { return Objects.equals(this.id, that.id) && Objects.equals(this.source, that.source) && Objects.equals(this.dest, that.dest) + && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.headers, that.headers) && Objects.equals(this.pivotConfig, that.pivotConfig) && Objects.equals(this.description, that.description); @@ -225,7 +257,7 @@ public boolean equals(Object other) { @Override public int hashCode(){ - return Objects.hash(id, source, dest, headers, pivotConfig, description); + return Objects.hash(id, source, dest, syncConfig, headers, pivotConfig, description); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java new file mode 100644 index 0000000000000..d8008f12126e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/SyncConfig.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.index.query.QueryBuilder; + +public interface SyncConfig extends ToXContentObject, NamedWriteable { + + /** + * Validate configuration + * + * @return true if valid + */ + boolean isValid(); + + QueryBuilder getBoundaryQuery(DataFrameTransformCheckpoint checkpoint); + + QueryBuilder getChangesQuery(DataFrameTransformCheckpoint oldCheckpoint, DataFrameTransformCheckpoint newCheckpoint); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java new file mode 100644 index 0000000000000..9a949ab21e8c3 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfig.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TimeSyncConfig implements SyncConfig { + + private static final String NAME = "data_frame_transform_pivot_sync_time"; + + private final String field; + private final TimeValue delay; + + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + + private static ConstructingObjectParser createParser(boolean lenient) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, + args -> { + String field = (String) args[0]; + TimeValue delay = args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO; + + return new TimeSyncConfig(field, delay); + }); + + parser.declareString(constructorArg(), DataFrameField.FIELD); + parser.declareField(optionalConstructorArg(), + (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), DataFrameField.DELAY.getPreferredName()), DataFrameField.DELAY, + ObjectParser.ValueType.STRING_OR_NULL); + + return parser; + } + + public TimeSyncConfig() { + this(null, null); + } + + public TimeSyncConfig(final String field, final TimeValue delay) { + this.field = ExceptionsHelper.requireNonNull(field, DataFrameField.FIELD.getPreferredName()); + this.delay = ExceptionsHelper.requireNonNull(delay, DataFrameField.DELAY.getPreferredName()); + } + + public TimeSyncConfig(StreamInput in) throws IOException { + this.field = in.readString(); + this.delay = in.readTimeValue(); + } + + public String getField() { + return field; + } + + public TimeValue getDelay() { + return delay; + } + + @Override + public boolean isValid() { + return true; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(field); + out.writeTimeValue(delay); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(DataFrameField.FIELD.getPreferredName(), field); + if (delay.duration() > 0) { + builder.field(DataFrameField.DELAY.getPreferredName(), delay.getStringRep()); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeSyncConfig that = (TimeSyncConfig) other; + + return Objects.equals(this.field, that.field) + && Objects.equals(this.delay, that.delay); + } + + @Override + public int hashCode(){ + return Objects.hash(field, delay); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + + public static TimeSyncConfig parse(final XContentParser parser) { + return LENIENT_PARSER.apply(parser, null); + } + + public static TimeSyncConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { + return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); + } + + @Override + public String getWriteableName() { + return DataFrameField.TIME_BASED_SYNC.getPreferredName(); + } + + @Override + public QueryBuilder getBoundaryQuery(DataFrameTransformCheckpoint checkpoint) { + return new RangeQueryBuilder(field).lt(checkpoint.getTimeUpperBound()).format("epoch_millis"); + } + + @Override + public QueryBuilder getChangesQuery(DataFrameTransformCheckpoint oldCheckpoint, DataFrameTransformCheckpoint newCheckpoint) { + return new RangeQueryBuilder(field).gte(oldCheckpoint.getTimeUpperBound()).lt(newCheckpoint.getTimeUpperBound()) + .format("epoch_millis"); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java index 0cfc659e50646..0e0187d8ef02b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.dataframe.action.PreviewDataFrameTransformAction.Request; import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests; import org.elasticsearch.xpack.core.dataframe.transforms.DestConfig; import org.elasticsearch.xpack.core.dataframe.transforms.pivot.PivotConfigTests; @@ -39,9 +40,14 @@ protected boolean supportsUnknownFields() { @Override protected Request createTestInstance() { - DataFrameTransformConfig config = new DataFrameTransformConfig("transform-preview", randomSourceConfig(), + DataFrameTransformConfig config = new DataFrameTransformConfig( + "transform-preview", + randomSourceConfig(), new DestConfig("unused-transform-preview-index"), - null, PivotConfigTests.randomPivotConfig(), null); + randomBoolean() ? DataFrameTransformConfigTests.randomSyncConfig() : null, + null, + PivotConfigTests.randomPivotConfig(), + null); return new Request(config); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java index a735b5a02acb8..8b46b8cd2838a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfigTests.java @@ -41,24 +41,28 @@ public static DataFrameTransformConfig randomDataFrameTransformConfig() { } public static DataFrameTransformConfig randomDataFrameTransformConfigWithoutHeaders(String id) { - return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), null, + return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomBoolean() ? randomSyncConfig() : null, null, PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); } public static DataFrameTransformConfig randomDataFrameTransformConfig(String id) { - return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomHeaders(), - PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); + return new DataFrameTransformConfig(id, randomSourceConfig(), randomDestConfig(), randomBoolean() ? randomSyncConfig() : null, + randomHeaders(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); } public static DataFrameTransformConfig randomInvalidDataFrameTransformConfig() { if (randomBoolean()) { - return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(), - randomDestConfig(), randomHeaders(), PivotConfigTests.randomPivotConfig(), - randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(), randomDestConfig(), + randomBoolean() ? randomSyncConfig() : null, randomHeaders(), PivotConfigTests.randomPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); } // else - return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), - randomDestConfig(), randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(), - randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100)); + return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(), randomDestConfig(), + randomBoolean() ? randomSyncConfig() : null, randomHeaders(), PivotConfigTests.randomInvalidPivotConfig(), + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000)); + } + + public static SyncConfig randomSyncConfig() { + return TimeSyncConfigTests.randomTimeSyncConfig(); } @Before @@ -167,11 +171,11 @@ public void testXContentForInternalStorage() throws IOException { public void testMaxLengthDescription() { IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> new DataFrameTransformConfig("id", - randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001))); + randomSourceConfig(), randomDestConfig(), null, null, PivotConfigTests.randomPivotConfig(), randomAlphaOfLength(1001))); assertThat(exception.getMessage(), equalTo("[description] must be less than 1000 characters in length.")); String description = randomAlphaOfLength(1000); DataFrameTransformConfig config = new DataFrameTransformConfig("id", - randomSourceConfig(), randomDestConfig(), null, PivotConfigTests.randomPivotConfig(), description); + randomSourceConfig(), randomDestConfig(), null, null, PivotConfigTests.randomPivotConfig(), description); assertThat(description, equalTo(config.getDescription())); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java new file mode 100644 index 0000000000000..763e13e77aee0 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/TimeSyncConfigTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.dataframe.transforms; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; + +import java.io.IOException; + +public class TimeSyncConfigTests extends AbstractSerializingTestCase { + + public static TimeSyncConfig randomTimeSyncConfig() { + return new TimeSyncConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + } + + @Override + protected TimeSyncConfig doParseInstance(XContentParser parser) throws IOException { + return TimeSyncConfig.fromXContent(parser, false); + } + + @Override + protected TimeSyncConfig createTestInstance() { + return randomTimeSyncConfig(); + } + + @Override + protected Reader instanceReader() { + return TimeSyncConfig::new; + } + +} diff --git a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java index 84f3e05de5cd1..0eeaf6d405298 100644 --- a/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java +++ b/x-pack/plugin/data-frame/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameIntegTestCase.java @@ -192,6 +192,7 @@ protected DataFrameTransformConfig createTransformConfig(String id, return new DataFrameTransformConfig(id, new SourceConfig(sourceIndices, createQueryConfig(queryBuilder)), new DestConfig(destinationIndex), + null, Collections.emptyMap(), createPivotConfig(groups, aggregations), "Test data frame transform config id: " + id); diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java index d338d6949f07b..ec48c10ee6c9a 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameTransformProgressIT.java @@ -135,6 +135,7 @@ public void testGetProgress() throws Exception { sourceConfig, destConfig, null, + null, pivotConfig, null); @@ -155,6 +156,7 @@ public void testGetProgress() throws Exception { sourceConfig, destConfig, null, + null, pivotConfig, null); diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java index b7e6c235f8e6c..75c8c7443bd72 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/DataFrame.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.settings.SettingsModule; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.license.XPackLicenseState; @@ -40,6 +41,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider; import org.elasticsearch.xpack.core.dataframe.action.DeleteDataFrameTransformAction; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsAction; import org.elasticsearch.xpack.core.dataframe.action.GetDataFrameTransformsStatsAction; @@ -231,4 +233,9 @@ public void close() { schedulerEngine.get().stop(); } } + + @Override + public List getNamedXContent() { + return new DataFrameNamedXContentProvider().getNamedXContentParsers(); + } } From 5cd976a087eecf5a2acc6705a861684c2132b6b1 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 3 May 2019 17:47:15 +0200 Subject: [PATCH 2/5] remove merge file --- .../client/RestHighLevelClientTests.java.orig | 965 ------------------ 1 file changed, 965 deletions(-) delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig deleted file mode 100644 index f37ef72a547ca..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java.orig +++ /dev/null @@ -1,965 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import com.fasterxml.jackson.core.JsonParseException; - -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.ProtocolVersion; -import org.apache.http.RequestLine; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicHttpResponse; -import org.apache.http.message.BasicRequestLine; -import org.apache.http.message.BasicStatusLine; -import org.apache.http.nio.entity.NByteArrayEntity; -import org.apache.http.nio.entity.NStringEntity; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchResponseSections; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.client.core.MainRequest; -import org.elasticsearch.client.core.MainResponse; -import org.elasticsearch.client.dataframe.transforms.SyncConfig; -import org.elasticsearch.client.dataframe.transforms.TimeSyncConfig; -import org.elasticsearch.client.indexlifecycle.AllocateAction; -import org.elasticsearch.client.indexlifecycle.DeleteAction; -import org.elasticsearch.client.indexlifecycle.ForceMergeAction; -import org.elasticsearch.client.indexlifecycle.FreezeAction; -import org.elasticsearch.client.indexlifecycle.LifecycleAction; -import org.elasticsearch.client.indexlifecycle.ReadOnlyAction; -import org.elasticsearch.client.indexlifecycle.RolloverAction; -import org.elasticsearch.client.indexlifecycle.SetPriorityAction; -import org.elasticsearch.client.indexlifecycle.ShrinkAction; -import org.elasticsearch.client.indexlifecycle.UnfollowAction; -import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis; -import org.elasticsearch.client.ml.dataframe.OutlierDetection; -import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContentFragment; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.cbor.CborXContent; -import org.elasticsearch.common.xcontent.smile.SmileXContent; -import org.elasticsearch.index.rankeval.DiscountedCumulativeGain; -import org.elasticsearch.index.rankeval.EvaluationMetric; -import org.elasticsearch.index.rankeval.ExpectedReciprocalRank; -import org.elasticsearch.index.rankeval.MeanReciprocalRank; -import org.elasticsearch.index.rankeval.MetricDetail; -import org.elasticsearch.index.rankeval.PrecisionAtK; -import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; -import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; -import org.hamcrest.Matchers; -import org.junit.Before; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.net.SocketTimeoutException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; -import static org.hamcrest.CoreMatchers.endsWith; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class RestHighLevelClientTests extends ESTestCase { - - private static final String SUBMIT_TASK_PREFIX = "submit_"; - private static final String SUBMIT_TASK_SUFFIX = "_task"; - private static final ProtocolVersion HTTP_PROTOCOL = new ProtocolVersion("http", 1, 1); - private static final RequestLine REQUEST_LINE = new BasicRequestLine(HttpGet.METHOD_NAME, "/", HTTP_PROTOCOL); - - /** - * These APIs do not use a Request object (because they don't have a body, or any request parameters). - * The method naming/parameter assertions use this {@code Set} to determine which rules to apply. - * (This is also used for async variants of these APIs when they exist) - */ - private static final Set APIS_WITHOUT_REQUEST_OBJECT = Sets.newHashSet( - // core - "ping", "info", - // security - "security.get_ssl_certificates", "security.authenticate", "security.get_user_privileges", - // license - "license.get_trial_status", "license.get_basic_status" - - ); - - private RestClient restClient; - private RestHighLevelClient restHighLevelClient; - - @Before - public void initClient() { - restClient = mock(RestClient.class); - restHighLevelClient = new RestHighLevelClient(restClient, RestClient::close, Collections.emptyList()); - } - - public void testCloseIsIdempotent() throws IOException { - restHighLevelClient.close(); - verify(restClient, times(1)).close(); - restHighLevelClient.close(); - verify(restClient, times(2)).close(); - restHighLevelClient.close(); - verify(restClient, times(3)).close(); - } - - public void testPingSuccessful() throws IOException { - Response response = mock(Response.class); - when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.OK)); - when(restClient.performRequest(any(Request.class))).thenReturn(response); - assertTrue(restHighLevelClient.ping(RequestOptions.DEFAULT)); - } - - public void testPing404NotFound() throws IOException { - Response response = mock(Response.class); - when(response.getStatusLine()).thenReturn(newStatusLine(RestStatus.NOT_FOUND)); - when(restClient.performRequest(any(Request.class))).thenReturn(response); - assertFalse(restHighLevelClient.ping(RequestOptions.DEFAULT)); - } - - public void testPingSocketTimeout() throws IOException { - when(restClient.performRequest(any(Request.class))).thenThrow(new SocketTimeoutException()); - expectThrows(SocketTimeoutException.class, () -> restHighLevelClient.ping(RequestOptions.DEFAULT)); - } - - public void testInfo() throws IOException { - MainResponse testInfo = new MainResponse("nodeName", new MainResponse.Version("number", "buildFlavor", "buildType", "buildHash", - "buildDate", true, "luceneVersion", "minimumWireCompatibilityVersion", "minimumIndexCompatibilityVersion"), - "clusterName", "clusterUuid", "You Know, for Search"); - mockResponse((ToXContentFragment) (builder, params) -> { - // taken from the server side MainResponse - builder.field("name", testInfo.getNodeName()); - builder.field("cluster_name", testInfo.getClusterName()); - builder.field("cluster_uuid", testInfo.getClusterUuid()); - builder.startObject("version") - .field("number", testInfo.getVersion().getNumber()) - .field("build_flavor", testInfo.getVersion().getBuildFlavor()) - .field("build_type", testInfo.getVersion().getBuildType()) - .field("build_hash", testInfo.getVersion().getBuildHash()) - .field("build_date", testInfo.getVersion().getBuildDate()) - .field("build_snapshot", testInfo.getVersion().isSnapshot()) - .field("lucene_version", testInfo.getVersion().getLuceneVersion()) - .field("minimum_wire_compatibility_version", testInfo.getVersion().getMinimumWireCompatibilityVersion()) - .field("minimum_index_compatibility_version", testInfo.getVersion().getMinimumIndexCompatibilityVersion()) - .endObject(); - builder.field("tagline", testInfo.getTagline()); - return builder; - }); - MainResponse receivedInfo = restHighLevelClient.info(RequestOptions.DEFAULT); - assertEquals(testInfo, receivedInfo); - } - - public void testSearchScroll() throws IOException { - SearchResponse mockSearchResponse = new SearchResponse(new SearchResponseSections(SearchHits.empty(), InternalAggregations.EMPTY, - null, false, false, null, 1), randomAlphaOfLengthBetween(5, 10), 5, 5, 0, 100, ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY); - mockResponse(mockSearchResponse); - SearchResponse searchResponse = restHighLevelClient.scroll( - new SearchScrollRequest(randomAlphaOfLengthBetween(5, 10)), RequestOptions.DEFAULT); - assertEquals(mockSearchResponse.getScrollId(), searchResponse.getScrollId()); - assertEquals(0, searchResponse.getHits().getTotalHits().value); - assertEquals(5, searchResponse.getTotalShards()); - assertEquals(5, searchResponse.getSuccessfulShards()); - assertEquals(100, searchResponse.getTook().getMillis()); - } - - public void testClearScroll() throws IOException { - ClearScrollResponse mockClearScrollResponse = new ClearScrollResponse(randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE)); - mockResponse(mockClearScrollResponse); - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(randomAlphaOfLengthBetween(5, 10)); - ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - assertEquals(mockClearScrollResponse.isSucceeded(), clearScrollResponse.isSucceeded()); - assertEquals(mockClearScrollResponse.getNumFreed(), clearScrollResponse.getNumFreed()); - } - - private void mockResponse(ToXContent toXContent) throws IOException { - Response response = mock(Response.class); - ContentType contentType = ContentType.parse(RequestConverters.REQUEST_BODY_CONTENT_TYPE.mediaType()); - String requestBody = toXContent(toXContent, RequestConverters.REQUEST_BODY_CONTENT_TYPE, false).utf8ToString(); - when(response.getEntity()).thenReturn(new NStringEntity(requestBody, contentType)); - when(restClient.performRequest(any(Request.class))).thenReturn(response); - } - - public void testRequestValidation() { - ActionRequestValidationException validationException = new ActionRequestValidationException(); - validationException.addValidationError("validation error"); - ActionRequest request = new ActionRequest() { - @Override - public ActionRequestValidationException validate() { - return validationException; - } - }; - - { - ActionRequestValidationException actualException = expectThrows(ActionRequestValidationException.class, - () -> restHighLevelClient.performRequest(request, null, RequestOptions.DEFAULT, null, null)); - assertSame(validationException, actualException); - } - { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - restHighLevelClient.performRequestAsync(request, null, RequestOptions.DEFAULT, null, trackingActionListener, null); - assertSame(validationException, trackingActionListener.exception.get()); - } - } - - public void testParseEntity() throws IOException { - { - IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(null, null)); - assertEquals("Response body expected but not returned", ise.getMessage()); - } - { - IllegalStateException ise = expectThrows(IllegalStateException.class, - () -> restHighLevelClient.parseEntity(new NStringEntity("", (ContentType) null), null)); - assertEquals("Elasticsearch didn't return the [Content-Type] header, unable to parse response body", ise.getMessage()); - } - { - NStringEntity entity = new NStringEntity("", ContentType.APPLICATION_SVG_XML); - IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(entity, null)); - assertEquals("Unsupported Content-Type: " + entity.getContentType().getValue(), ise.getMessage()); - } - { - CheckedFunction entityParser = parser -> { - assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); - assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); - assertTrue(parser.nextToken().isValue()); - String value = parser.text(); - assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); - return value; - }; - HttpEntity jsonEntity = new NStringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON); - assertEquals("value", restHighLevelClient.parseEntity(jsonEntity, entityParser)); - HttpEntity yamlEntity = new NStringEntity("---\nfield: value\n", ContentType.create("application/yaml")); - assertEquals("value", restHighLevelClient.parseEntity(yamlEntity, entityParser)); - HttpEntity smileEntity = createBinaryEntity(SmileXContent.contentBuilder(), ContentType.create("application/smile")); - assertEquals("value", restHighLevelClient.parseEntity(smileEntity, entityParser)); - HttpEntity cborEntity = createBinaryEntity(CborXContent.contentBuilder(), ContentType.create("application/cbor")); - assertEquals("value", restHighLevelClient.parseEntity(cborEntity, entityParser)); - } - } - - private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, ContentType contentType) throws IOException { - try (XContentBuilder builder = xContentBuilder) { - builder.startObject(); - builder.field("field", "value"); - builder.endObject(); - return new NByteArrayEntity(BytesReference.bytes(builder).toBytesRef().bytes, contentType); - } - } - - public void testConvertExistsResponse() { - RestStatus restStatus = randomBoolean() ? RestStatus.OK : randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - boolean result = RestHighLevelClient.convertExistsResponse(response); - assertEquals(restStatus == RestStatus.OK, result); - } - - public void testParseResponseException() throws IOException { - { - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); - assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - } - { - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", - ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); - assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getSuppressed()[0]); - } - { - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IOException.class)); - } - { - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - ElasticsearchException elasticsearchException = restHighLevelClient.parseResponseException(responseException); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); - } - } - - public void testPerformRequestOnSuccess() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - when(restClient.performRequest(any(Request.class))).thenReturn(mockResponse); - { - Integer result = restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.emptySet()); - assertEquals(restStatus.getStatus(), result.intValue()); - } - { - IOException ioe = expectThrows(IOException.class, () -> restHighLevelClient.performRequest(mainRequest, - requestConverter, RequestOptions.DEFAULT, response -> {throw new IllegalStateException();}, Collections.emptySet())); - assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " + - "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage()); - } - } - - public void testPerformRequestOnResponseExceptionWithoutEntity() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); - assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - } - - public void testPerformRequestOnResponseExceptionWithEntity() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", - ContentType.APPLICATION_JSON)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); - assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getSuppressed()[0]); - } - - public void testPerformRequestOnResponseExceptionWithBrokenEntity() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class)); - } - - public void testPerformRequestOnResponseExceptionWithBrokenEntity2() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); - } - - public void testPerformRequestOnResponseExceptionWithIgnores() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - //although we got an exception, we turn it into a successful response because the status code was provided among ignores - assertEquals(Integer.valueOf(404), restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> response.getStatusLine().getStatusCode(), Collections.singleton(404))); - } - - public void testPerformRequestOnResponseExceptionWithIgnoresErrorNoBody() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> {throw new IllegalStateException();}, Collections.singleton(404))); - assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); - } - - public void testPerformRequestOnResponseExceptionWithIgnoresErrorValidBody() throws IOException { - MainRequest mainRequest = new MainRequest(); - CheckedFunction requestConverter = request -> new Request(HttpGet.METHOD_NAME, "/"); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":404}", - ContentType.APPLICATION_JSON)); - Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(mockResponse); - when(restClient.performRequest(any(Request.class))).thenThrow(responseException); - ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, - response -> {throw new IllegalStateException();}, Collections.singleton(404))); - assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getSuppressed()[0]); - assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); - } - - public void testWrapResponseListenerOnSuccess() { - { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse)); - assertNull(trackingActionListener.exception.get()); - assertEquals(restStatus.getStatus(), trackingActionListener.statusCode.get()); - } - { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> {throw new IllegalStateException();}, trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - responseListener.onSuccess(new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse)); - assertThat(trackingActionListener.exception.get(), instanceOf(IOException.class)); - IOException ioe = (IOException) trackingActionListener.exception.get(); - assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " + - "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage()); - assertThat(ioe.getCause(), instanceOf(IllegalStateException.class)); - } - } - - public void testWrapResponseListenerOnException() { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - IllegalStateException exception = new IllegalStateException(); - responseListener.onFailure(exception); - assertSame(exception, trackingActionListener.exception.get()); - } - - public void testWrapResponseListenerOnResponseExceptionWithoutEntity() throws IOException { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException) trackingActionListener.exception.get(); - assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - } - - public void testWrapResponseListenerOnResponseExceptionWithEntity() throws IOException { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":" + restStatus.getStatus() + "}", - ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); - assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getSuppressed()[0]); - } - - public void testWrapResponseListenerOnResponseExceptionWithBrokenEntity() throws IOException { - { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"error\":", ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(JsonParseException.class)); - } - { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.emptySet()); - RestStatus restStatus = randomFrom(RestStatus.values()); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(restStatus)); - httpResponse.setEntity(new NStringEntity("{\"status\":" + restStatus.getStatus() + "}", ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); - assertEquals("Unable to parse response body", elasticsearchException.getMessage()); - assertEquals(restStatus, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertThat(elasticsearchException.getSuppressed()[0], instanceOf(IllegalStateException.class)); - } - } - - public void testWrapResponseListenerOnResponseExceptionWithIgnores() throws IOException { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> response.getStatusLine().getStatusCode(), trackingActionListener, Collections.singleton(404)); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - //although we got an exception, we turn it into a successful response because the status code was provided among ignores - assertNull(trackingActionListener.exception.get()); - assertEquals(404, trackingActionListener.statusCode.get()); - } - - public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorNoBody() throws IOException { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying - //to parse a 404 response which contains an error rather than a valid document not found response. - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404)); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); - assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getCause()); - assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); - } - - public void testWrapResponseListenerOnResponseExceptionWithIgnoresErrorValidBody() throws IOException { - TrackingActionListener trackingActionListener = new TrackingActionListener(); - //response parsing throws exception while handling ignores. same as when GetResponse#fromXContent throws error when trying - //to parse a 404 response which contains an error rather than a valid document not found response. - ResponseListener responseListener = restHighLevelClient.wrapResponseListener( - response -> { throw new IllegalStateException(); }, trackingActionListener, Collections.singleton(404)); - HttpResponse httpResponse = new BasicHttpResponse(newStatusLine(RestStatus.NOT_FOUND)); - httpResponse.setEntity(new NStringEntity("{\"error\":\"test error message\",\"status\":404}", - ContentType.APPLICATION_JSON)); - Response response = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); - ResponseException responseException = new ResponseException(response); - responseListener.onFailure(responseException); - assertThat(trackingActionListener.exception.get(), instanceOf(ElasticsearchException.class)); - ElasticsearchException elasticsearchException = (ElasticsearchException)trackingActionListener.exception.get(); - assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); - assertSame(responseException, elasticsearchException.getSuppressed()[0]); - assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); - } - - public void testDefaultNamedXContents() { - List namedXContents = RestHighLevelClient.getDefaultNamedXContents(); - int expectedInternalAggregations = InternalAggregationTestCase.getDefaultNamedXContents().size(); - int expectedSuggestions = 3; - assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size()); - Map, Integer> categories = new HashMap<>(); - for (NamedXContentRegistry.Entry namedXContent : namedXContents) { - Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1); - if (counter != null) { - categories.put(namedXContent.categoryClass, counter + 1); - } - } - assertEquals(2, categories.size()); - assertEquals(expectedInternalAggregations, categories.get(Aggregation.class).intValue()); - assertEquals(expectedSuggestions, categories.get(Suggest.Suggestion.class).intValue()); - } - - public void testProvidedNamedXContents() { - List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); - assertEquals(21, namedXContents.size()); - Map, Integer> categories = new HashMap<>(); - List names = new ArrayList<>(); - for (NamedXContentRegistry.Entry namedXContent : namedXContents) { - names.add(namedXContent.name.getPreferredName()); - Integer counter = categories.putIfAbsent(namedXContent.categoryClass, 1); - if (counter != null) { - categories.put(namedXContent.categoryClass, counter + 1); - } - } - assertEquals("Had: " + categories, 5, categories.size()); - assertEquals(Integer.valueOf(3), categories.get(Aggregation.class)); - assertTrue(names.contains(ChildrenAggregationBuilder.NAME)); - assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME)); - assertEquals(Integer.valueOf(4), categories.get(EvaluationMetric.class)); - assertTrue(names.contains(PrecisionAtK.NAME)); - assertTrue(names.contains(DiscountedCumulativeGain.NAME)); - assertTrue(names.contains(MeanReciprocalRank.NAME)); - assertTrue(names.contains(ExpectedReciprocalRank.NAME)); - assertEquals(Integer.valueOf(4), categories.get(MetricDetail.class)); - assertTrue(names.contains(PrecisionAtK.NAME)); - assertTrue(names.contains(MeanReciprocalRank.NAME)); - assertTrue(names.contains(DiscountedCumulativeGain.NAME)); - assertTrue(names.contains(ExpectedReciprocalRank.NAME)); - assertEquals(Integer.valueOf(9), categories.get(LifecycleAction.class)); - assertTrue(names.contains(UnfollowAction.NAME)); - assertTrue(names.contains(AllocateAction.NAME)); - assertTrue(names.contains(DeleteAction.NAME)); - assertTrue(names.contains(ForceMergeAction.NAME)); - assertTrue(names.contains(ReadOnlyAction.NAME)); - assertTrue(names.contains(RolloverAction.NAME)); - assertTrue(names.contains(ShrinkAction.NAME)); - assertTrue(names.contains(FreezeAction.NAME)); - assertTrue(names.contains(SetPriorityAction.NAME)); -<<<<<<< HEAD - assertEquals(Integer.valueOf(1), categories.get(DataFrameAnalysis.class)); - assertTrue(names.contains(OutlierDetection.NAME.getPreferredName())); -======= - assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class)); - assertTrue(names.contains(TimeSyncConfig.NAME)); ->>>>>>> d94764e95aa... introduce sync API - } - - public void testApiNamingConventions() throws Exception { - //this list should be empty once the high-level client is feature complete - String[] notYetSupportedApi = new String[]{ - "cluster.remote_info", - "create", - "get_source", - "indices.delete_alias", - "indices.exists_type", - "indices.get_upgrade", - "indices.put_alias", - "render_search_template", - "scripts_painless_execute" - }; - //These API are not required for high-level client feature completeness - String[] notRequiredApi = new String[] { - "cluster.allocation_explain", - "cluster.pending_tasks", - "cluster.reroute", - "cluster.state", - "cluster.stats", - "indices.shard_stores", - "indices.upgrade", - "indices.recovery", - "indices.segments", - "indices.stats", - "ingest.processor_grok", - "nodes.info", - "nodes.stats", - "nodes.hot_threads", - "nodes.usage", - "nodes.reload_secure_settings", - "scripts_painless_context", - "search_shards", - }; - List booleanReturnMethods = Arrays.asList( - "security.enable_user", - "security.disable_user", - "security.change_password"); - Set deprecatedMethods = new HashSet<>(); - deprecatedMethods.add("indices.force_merge"); - deprecatedMethods.add("multi_get"); - deprecatedMethods.add("multi_search"); - deprecatedMethods.add("search_scroll"); - - ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load("/rest-api-spec/api"); - Set apiSpec = restSpec.getApis().stream().map(ClientYamlSuiteRestApi::getName).collect(Collectors.toSet()); - Set apiUnsupported = new HashSet<>(apiSpec); - Set apiNotFound = new HashSet<>(); - - Set topLevelMethodsExclusions = new HashSet<>(); - topLevelMethodsExclusions.add("getLowLevelClient"); - topLevelMethodsExclusions.add("close"); - - Map> methods = Arrays.stream(RestHighLevelClient.class.getMethods()) - .filter(method -> method.getDeclaringClass().equals(RestHighLevelClient.class) - && topLevelMethodsExclusions.contains(method.getName()) == false) - .map(method -> Tuple.tuple(toSnakeCase(method.getName()), method)) - .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client") - ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple)) - .filter(tuple -> tuple.v2().getAnnotation(Deprecated.class) == null) - .collect(Collectors.groupingBy(Tuple::v1, - Collectors.mapping(Tuple::v2, Collectors.toSet()))); - - // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate - // impl will replace the existing getTemplate method. - // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` - // looking like it doesn't have a valid implementatation when it does. - apiUnsupported.remove("indices.get_template"); - - - - for (Map.Entry> entry : methods.entrySet()) { - String apiName = entry.getKey(); - - for (Method method : entry.getValue()) { - assertTrue("method [" + apiName + "] is not final", - Modifier.isFinal(method.getClass().getModifiers()) || Modifier.isFinal(method.getModifiers())); - assertTrue("method [" + method + "] should be public", Modifier.isPublic(method.getModifiers())); - - //we convert all the method names to snake case, hence we need to look for the '_async' suffix rather than 'Async' - if (apiName.endsWith("_async")) { - assertAsyncMethod(methods, method, apiName); - } else if (isSubmitTaskMethod(apiName)) { - assertSubmitTaskMethod(methods, method, apiName, restSpec); - } else { - assertSyncMethod(method, apiName, booleanReturnMethods); - apiUnsupported.remove(apiName); - if (apiSpec.contains(apiName) == false) { - if (deprecatedMethods.contains(apiName)) { - assertTrue("method [" + method.getName() + "], api [" + apiName + "] should be deprecated", - method.isAnnotationPresent(Deprecated.class)); - } else { - //TODO xpack api are currently ignored, we need to load xpack yaml spec too - if (apiName.startsWith("xpack.") == false && - apiName.startsWith("license.") == false && - apiName.startsWith("machine_learning.") == false && - apiName.startsWith("rollup.") == false && - apiName.startsWith("watcher.") == false && - apiName.startsWith("graph.") == false && - apiName.startsWith("migration.") == false && - apiName.startsWith("security.") == false && - apiName.startsWith("index_lifecycle.") == false && - apiName.startsWith("ccr.") == false && - apiName.startsWith("data_frame") == false && - apiName.endsWith("freeze") == false && - // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we - // can get rid of 7.0's deprecated "getTemplate" - apiName.equals("indices.get_index_template") == false) { - apiNotFound.add(apiName); - } - } - } - } - } - } - assertThat("Some client method doesn't match a corresponding API defined in the REST spec: " + apiNotFound, - apiNotFound.size(), equalTo(0)); - - //we decided not to support cat API in the high-level REST client, they are supposed to be used from a low-level client - apiUnsupported.removeIf(api -> api.startsWith("cat.")); - Stream.concat(Arrays.stream(notYetSupportedApi), Arrays.stream(notRequiredApi)).forEach( - api -> assertTrue(api + " API is either not defined in the spec or already supported by the high-level client", - apiUnsupported.remove(api))); - assertThat("Some API are not supported but they should be: " + apiUnsupported, apiUnsupported.size(), equalTo(0)); - } - - private static void assertSyncMethod(Method method, String apiName, List booleanReturnMethods) { - //A few methods return a boolean rather than a response object - if (apiName.equals("ping") || apiName.contains("exist") || booleanReturnMethods.contains(apiName)) { - assertThat("the return type for method [" + method + "] is incorrect", - method.getReturnType().getSimpleName(), equalTo("boolean")); - } else { - // It's acceptable for 404s to be represented as empty Optionals - if (!method.getReturnType().isAssignableFrom(Optional.class)) { - assertThat("the return type for method [" + method + "] is incorrect", - method.getReturnType().getSimpleName(), endsWith("Response")); - } - } - - assertEquals("incorrect number of exceptions for method [" + method + "]", 1, method.getExceptionTypes().length); - //a few methods don't accept a request object as argument - if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName)) { - assertEquals("incorrect number of arguments for method [" + method + "]", 1, method.getParameterTypes().length); - assertThat("the parameter to method [" + method + "] is the wrong type", - method.getParameterTypes()[0], equalTo(RequestOptions.class)); - } else { - assertEquals("incorrect number of arguments for method [" + method + "]", 2, method.getParameterTypes().length); - // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation - if (method.getParameterTypes()[0].equals(RequestOptions.class)) { - assertThat("the first parameter to method [" + method + "] is the wrong type", - method.getParameterTypes()[0], equalTo(RequestOptions.class)); - assertThat("the second parameter to method [" + method + "] is the wrong type", - method.getParameterTypes()[1].getSimpleName(), endsWith("Request")); - } else { - assertThat("the first parameter to method [" + method + "] is the wrong type", - method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); - assertThat("the second parameter to method [" + method + "] is the wrong type", - method.getParameterTypes()[1], equalTo(RequestOptions.class)); - } - } - } - - private static void assertAsyncMethod(Map> methods, Method method, String apiName) { - assertTrue("async method [" + method.getName() + "] doesn't have corresponding sync method", - methods.containsKey(apiName.substring(0, apiName.length() - 6))); - assertThat("async method [" + method + "] should return void", method.getReturnType(), equalTo(Void.TYPE)); - assertEquals("async method [" + method + "] should not throw any exceptions", 0, method.getExceptionTypes().length); - if (APIS_WITHOUT_REQUEST_OBJECT.contains(apiName.replaceAll("_async$", ""))) { - assertEquals(2, method.getParameterTypes().length); - assertThat(method.getParameterTypes()[0], equalTo(RequestOptions.class)); - assertThat(method.getParameterTypes()[1], equalTo(ActionListener.class)); - } else { - assertEquals("async method [" + method + "] has the wrong number of arguments", 3, method.getParameterTypes().length); - // This is no longer true for all methods. Some methods can contain these 2 args backwards because of deprecation - if (method.getParameterTypes()[0].equals(RequestOptions.class)) { - assertThat("the first parameter to async method [" + method + "] should be a request type", - method.getParameterTypes()[0], equalTo(RequestOptions.class)); - assertThat("the second parameter to async method [" + method + "] is the wrong type", - method.getParameterTypes()[1].getSimpleName(), endsWith("Request")); - } else { - assertThat("the first parameter to async method [" + method + "] should be a request type", - method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); - assertThat("the second parameter to async method [" + method + "] is the wrong type", - method.getParameterTypes()[1], equalTo(RequestOptions.class)); - } - assertThat("the third parameter to async method [" + method + "] is the wrong type", - method.getParameterTypes()[2], equalTo(ActionListener.class)); - } - } - - private static void assertSubmitTaskMethod(Map> methods, Method method, String apiName, - ClientYamlSuiteRestSpec restSpec) { - String methodName = extractMethodName(apiName); - assertTrue("submit task method [" + method.getName() + "] doesn't have corresponding sync method", - methods.containsKey(methodName)); - assertEquals("submit task method [" + method + "] has the wrong number of arguments", 2, method.getParameterTypes().length); - assertThat("the first parameter to submit task method [" + method + "] is the wrong type", - method.getParameterTypes()[0].getSimpleName(), endsWith("Request")); - assertThat("the second parameter to submit task method [" + method + "] is the wrong type", - method.getParameterTypes()[1], equalTo(RequestOptions.class)); - - assertThat("submit task method [" + method + "] must have wait_for_completion parameter in rest spec", - restSpec.getApi(methodName).getParams(), Matchers.hasKey("wait_for_completion")); - } - - private static String extractMethodName(String apiName) { - return apiName.substring(SUBMIT_TASK_PREFIX.length(), apiName.length() - SUBMIT_TASK_SUFFIX.length()); - } - - private static boolean isSubmitTaskMethod(String apiName) { - return apiName.startsWith(SUBMIT_TASK_PREFIX) && apiName.endsWith(SUBMIT_TASK_SUFFIX); - } - - private static Stream> getSubClientMethods(String namespace, Class clientClass) { - return Arrays.stream(clientClass.getMethods()).filter(method -> method.getDeclaringClass().equals(clientClass)) - .map(method -> Tuple.tuple(namespace + "." + toSnakeCase(method.getName()), method)) - .flatMap(tuple -> tuple.v2().getReturnType().getName().endsWith("Client") - ? getSubClientMethods(tuple.v1(), tuple.v2().getReturnType()) : Stream.of(tuple)); - } - - private static String toSnakeCase(String camelCase) { - StringBuilder snakeCaseString = new StringBuilder(); - for (Character aChar : camelCase.toCharArray()) { - if (Character.isUpperCase(aChar)) { - snakeCaseString.append('_'); - snakeCaseString.append(Character.toLowerCase(aChar)); - } else { - snakeCaseString.append(aChar); - } - } - return snakeCaseString.toString(); - } - - private static class TrackingActionListener implements ActionListener { - private final AtomicInteger statusCode = new AtomicInteger(-1); - private final AtomicReference exception = new AtomicReference<>(); - - @Override - public void onResponse(Integer statusCode) { - assertTrue(this.statusCode.compareAndSet(-1, statusCode)); - } - - @Override - public void onFailure(Exception e) { - assertTrue(exception.compareAndSet(null, e)); - } - } - - private static StatusLine newStatusLine(RestStatus restStatus) { - return new BasicStatusLine(HTTP_PROTOCOL, restStatus.getStatus(), restStatus.name()); - } -} From f96531359c095096f45ff137f1fee68a224c00d8 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 3 May 2019 18:53:41 +0200 Subject: [PATCH 3/5] register XContent in tests --- .../action/AbstractSerializingDataFrameTestCase.java | 8 ++++++++ .../action/AbstractWireSerializingDataFrameTestCase.java | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java index 8b633cdfc26d5..14cbdef148ca4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractSerializingDataFrameTestCase.java @@ -13,6 +13,10 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider; +import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; import org.junit.Before; import java.util.List; @@ -30,7 +34,11 @@ public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); + namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(), + TimeSyncConfig::new)); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); namedXContentRegistry = new NamedXContentRegistry(namedXContents); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java index 91a7ec54dd256..47d7860b71da0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/AbstractWireSerializingDataFrameTestCase.java @@ -12,6 +12,10 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider; +import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig; +import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig; import org.junit.Before; import java.util.List; @@ -30,7 +34,11 @@ public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); + namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(), + TimeSyncConfig::new)); + List namedXContents = searchModule.getNamedXContents(); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); namedXContentRegistry = new NamedXContentRegistry(namedXContents); From e9eff4ff4c8cc53757bbe5fc9c1da245f1634b7a Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 3 May 2019 18:55:03 +0200 Subject: [PATCH 4/5] adapt hlrc test --- .../java/org/elasticsearch/client/RestHighLevelClientTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 5f4bb5ed5e0d7..6ca2e3c2bdd24 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -678,7 +678,7 @@ public void testProvidedNamedXContents() { categories.put(namedXContent.categoryClass, counter + 1); } } - assertEquals("Had: " + categories, 5, categories.size()); + assertEquals("Had: " + categories, 6, categories.size()); assertEquals(Integer.valueOf(3), categories.get(Aggregation.class)); assertTrue(names.contains(ChildrenAggregationBuilder.NAME)); assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME)); From d7c78dd07c85cb3fb8b5536f6dd62677be34652d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 3 May 2019 19:15:30 +0200 Subject: [PATCH 5/5] add named writables to abstractserializingtestcase --- .../transforms/AbstractSerializingDataFrameTestCase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java index 2b64fadac051a..79edb8084551d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/transforms/AbstractSerializingDataFrameTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.aggregations.BaseAggregationBuilder; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.dataframe.DataFrameField; +import org.elasticsearch.xpack.core.dataframe.DataFrameNamedXContentProvider; import org.junit.Before; import java.util.Collections; @@ -48,12 +49,15 @@ public void registerAggregationNamedObjects() throws Exception { MockDeprecatedQueryBuilder::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(AggregationBuilder.class, MockDeprecatedAggregationBuilder.NAME, MockDeprecatedAggregationBuilder::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(), + TimeSyncConfig::new)); List namedXContents = searchModule.getNamedXContents(); namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, new ParseField(MockDeprecatedQueryBuilder.NAME), (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p))); namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, new ParseField(MockDeprecatedAggregationBuilder.NAME), (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p))); + namedXContents.addAll(new DataFrameNamedXContentProvider().getNamedXContentParsers()); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); namedXContentRegistry = new NamedXContentRegistry(namedXContents);