From 27d5d30913a89897726dfe3d319051eed4fb79ce Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 17 Nov 2020 11:09:04 +0100 Subject: [PATCH 01/29] draft --- .../common/xcontent/XContentBuilder.java | 27 ++++++++++++------- .../rest/AbstractRestChannel.java | 6 ++++- .../elasticsearch/rest/BytesRestResponse.java | 2 +- .../AbstractXContentFilteringTestCase.java | 2 +- .../action/EnrichShardMultiSearchAction.java | 4 +-- .../exporter/FilteredMonitoringDoc.java | 2 +- .../action/activate/ActivateWatchTests.java | 2 +- .../watcher/common/http/HttpRequest.java | 9 +++---- .../notification/pagerduty/SentEvent.java | 4 +-- .../notification/slack/SentMessages.java | 4 +-- 10 files changed, 36 insertions(+), 26 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index e5aebb26fff4d..96f9a00325cdf 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -48,8 +48,6 @@ */ public final class XContentBuilder implements Closeable, Flushable { - private byte compatibleMajorVersion; - /** * Create a new {@link XContentBuilder} using the given {@link XContent} content. *

@@ -77,8 +75,8 @@ public static XContentBuilder builder(XContent xContent) throws IOException { * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. * @throws IOException if an {@link IOException} occurs while building the content */ - public static XContentBuilder builder(XContent xContent, Set includes, Set excludes) throws IOException { - return new XContentBuilder(xContent, new ByteArrayOutputStream(), includes, excludes); + public static XContentBuilder builder(XContentType xContent, Set includes, Set excludes) throws IOException { + return new XContentBuilder(xContent.xContent(), new ByteArrayOutputStream(), includes, excludes, xContent.mediaType()); } private static final Map, Writer> WRITERS; @@ -167,12 +165,16 @@ public interface HumanReadableTransformer { */ private boolean humanReadable = false; + private byte compatibleMajorVersion; + + private String responseContentTypeString; + /** * Constructs a new builder using the provided XContent and an OutputStream. Make sure * to call {@link #close()} when the builder is done with. */ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { - this(xContent, bos, Collections.emptySet(), Collections.emptySet()); + this(xContent, bos, Collections.emptySet(), Collections.emptySet(), null); } /** @@ -181,8 +183,8 @@ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { * filter will be written to the output stream. Make sure to call * {@link #close()} when the builder is done with. */ - public XContentBuilder(XContent xContent, OutputStream bos, Set includes) throws IOException { - this(xContent, bos, includes, Collections.emptySet()); + public XContentBuilder(XContentType xContentType, OutputStream bos, Set includes) throws IOException { + this(xContentType.xContent(), bos, includes, Collections.emptySet(), xContentType.mediaType()); } /** @@ -191,16 +193,21 @@ public XContentBuilder(XContent xContent, OutputStream bos, Set includes * remaining fields against the inclusive filters. *

* Make sure to call {@link #close()} when the builder is done with. - * - * @param os the output stream + * @param os the output stream * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. + * @param responseContentTypeString f */ - public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes) throws IOException { + public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, String responseContentTypeString) throws IOException { this.bos = os; + this.responseContentTypeString = responseContentTypeString; this.generator = xContent.createGenerator(bos, includes, excludes); } + public String getResponseContentTypeString() { + return responseContentTypeString; + } + public XContentType contentType() { return generator.contentType(); } diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index f08bf5b58e8b4..4755725f55c53 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -100,12 +100,15 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boo @Override public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nullable XContentType responseContentType, boolean useFiltering) throws IOException { + String responseContentTypeString = null; if (responseContentType == null) { if (Strings.hasText(format)) { responseContentType = XContentType.fromFormat(format); + responseContentTypeString = responseContentType.mediaType(); } if (responseContentType == null && Strings.hasText(acceptHeader)) { responseContentType = XContentType.fromMediaType(acceptHeader); + responseContentTypeString = acceptHeader; } } // try to determine the response content type from the media type or the format query string parameter, with the format parameter @@ -119,6 +122,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu // default to JSON output when all else fails responseContentType = XContentType.JSON; } + responseContentTypeString = responseContentType.mediaType(); } Set includes = Collections.emptySet(); @@ -131,7 +135,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); XContentBuilder builder = - new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes); + new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes, responseContentTypeString); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/server/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/server/src/main/java/org/elasticsearch/rest/BytesRestResponse.java index 028b3a5f8e693..99fa5805099dd 100644 --- a/server/src/main/java/org/elasticsearch/rest/BytesRestResponse.java +++ b/server/src/main/java/org/elasticsearch/rest/BytesRestResponse.java @@ -56,7 +56,7 @@ public class BytesRestResponse extends RestResponse { * Creates a new response based on {@link XContentBuilder}. */ public BytesRestResponse(RestStatus status, XContentBuilder builder) { - this(status, builder.contentType().mediaType(), BytesReference.bytes(builder)); + this(status, builder.getResponseContentTypeString(), BytesReference.bytes(builder)); } /** diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 4aa19b78a5ca0..b7a6fec26d318 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -54,7 +54,7 @@ private XContentBuilder createBuilder() throws IOException { } private XContentBuilder createBuilder(Set includes, Set excludes) throws IOException { - return XContentBuilder.builder(getXContentType().xContent(), includes, excludes); + return XContentBuilder.builder(getXContentType(), includes, excludes); } public void testSingleFieldObject() throws IOException { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 7cd1e85366989..22ffe831ea5f0 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -295,8 +295,8 @@ private static BytesReference filterSource(FetchSourceContext fetchSourceContext XContentType.SMILE.xContent(), new BytesStreamOutput(source.length()), includes, - excludes - ); + excludes, + XContentType.SMILE.mediaType()); XContentParser sourceParser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/FilteredMonitoringDoc.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/FilteredMonitoringDoc.java index 152e57fb531b6..9d44d23007fff 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/FilteredMonitoringDoc.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/FilteredMonitoringDoc.java @@ -58,7 +58,7 @@ Set getFilters() { public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { final XContent xContent = builder.contentType().xContent(); try (BytesStreamOutput out = new BytesStreamOutput()) { - try (XContentBuilder filteredBuilder = new XContentBuilder(xContent, out, filters)) { + try (XContentBuilder filteredBuilder = new XContentBuilder(builder.contentType(), out, filters)) { super.toXContent(filteredBuilder, params); } try (InputStream stream = out.bytes().streamInput(); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java index 8abc5ccd49ac6..4ec3d11e660a5 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/transport/action/activate/ActivateWatchTests.java @@ -137,7 +137,7 @@ public void testLoadWatchWithoutAState() throws Exception { "status.last_met_condition", "status.actions.**"); - XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), new BytesStreamOutput(), filters); + XContentBuilder builder = new XContentBuilder(XContentType.JSON, new BytesStreamOutput(), filters); source.toXContent(builder, ToXContent.EMPTY_PARAMS); // now that we filtered out the watch status state, lets put it back in diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java index 3cda915b7f336..2301dd1928b42 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -499,17 +498,17 @@ public interface Field { * Write a request via toXContent, but filter certain parts of it - this is needed to not expose secrets * * @param request The HttpRequest object to serialize - * @param xContent The xContent from the parent outputstream builder + * @param xContentType The XContentType from the parent outputstream builder * @param params The ToXContentParams from the parent write * @param excludeField The field to exclude * @return A bytearrayinputstream that contains the serialized request * @throws IOException if an IOException is triggered in the underlying toXContent method */ - public static InputStream filterToXContent(HttpRequest request, XContent xContent, ToXContent.Params params, + public static InputStream filterToXContent(HttpRequest request, XContentType xContentType, Params params, String excludeField) throws IOException { try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); - XContentBuilder filteredBuilder = new XContentBuilder(xContent, bos, - Collections.emptySet(), Collections.singleton(excludeField))) { + XContentBuilder filteredBuilder = new XContentBuilder(xContentType.xContent(), bos, + Collections.emptySet(), Collections.singleton(excludeField), xContentType.mediaType())) { request.toXContent(filteredBuilder, params); filteredBuilder.flush(); return new ByteArrayInputStream(bos.toByteArray()); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/SentEvent.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/SentEvent.java index 9bc040511a4a3..917804cebc537 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/SentEvent.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/pagerduty/SentEvent.java @@ -87,8 +87,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws // as this makes debugging pagerduty services much harder, this should be changed to only filter for // body.service_key - however the body is currently just a string, making filtering much harder if (WatcherParams.hideSecrets(params)) { - try (InputStream is = HttpRequest.filterToXContent(request, builder.contentType().xContent(), - params, "body")) { + try (InputStream is = HttpRequest.filterToXContent(request, builder.contentType(), + params, "body")) { builder.rawField(XField.REQUEST.getPreferredName(), is, builder.contentType()); } } else { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java index a9670c5ffd0ce..3ec00f33a826f 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java @@ -118,8 +118,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (WatcherParams.hideSecrets(params)) { // this writes out the request to the byte array output stream with the correct excludes // for slack - try (InputStream is = HttpRequest.filterToXContent(request, builder.contentType().xContent(), - params, "path")) { + try (InputStream is = HttpRequest.filterToXContent(request, builder.contentType(), + params, "path")) { builder.rawField(REQUEST.getPreferredName(), is, builder.contentType()); } } else { From 6e349a9bcfe0bc54f80fba98db8e9e6dac7fb367 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 17 Nov 2020 15:15:55 +0100 Subject: [PATCH 02/29] include in tests --- .../common/xcontent/XContentBuilder.java | 2 +- .../org/elasticsearch/rest/RestController.java | 8 +++++++- .../yaml/ClientYamlTestExecutionContext.java | 17 +++++++++++++++-- .../xpack/sql/plugin/RestSqlQueryAction.java | 4 +++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 96f9a00325cdf..154544d909e2a 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -174,7 +174,7 @@ public interface HumanReadableTransformer { * to call {@link #close()} when the builder is done with. */ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { - this(xContent, bos, Collections.emptySet(), Collections.emptySet(), null); + this(xContent, bos, Collections.emptySet(), Collections.emptySet(), xContent.type().mediaType()); } /** diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 10386d19a2030..cbf4957db53b9 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.path.PathTrie; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.internal.io.Streams; @@ -44,6 +45,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -175,7 +178,6 @@ private void registerHandlerNoWrap(RestRequest.Method method, String path, RestH final Version version = maybeWrappedHandler.compatibleWithVersion(); assert Version.CURRENT.minimumRestCompatibilityVersion() == version || Version.CURRENT == version : "REST API compatibility is only supported for version " + Version.CURRENT.minimumRestCompatibilityVersion().major; - handlers.insertOrUpdate(path, new MethodHandlers(path, maybeWrappedHandler, method), (mHandlers, newMHandler) -> mHandlers.addMethods(maybeWrappedHandler, method)); } @@ -328,8 +330,12 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel final String uri = request.uri(); final RestRequest.Method requestMethod; + Collection x = new ArrayList(); + Iterator iterator = x.iterator(); + Version compatibleVersion = this.compatibleVersion. get(request.getParsedAccept(), request.getParsedContentType(), request.hasContent()); + System.out.println(request.getParsedContentType()!=null ? request.getParsedContentType().getParameters().get(MediaType.COMPATIBLE_WITH_PARAMETER_NAME) : "null "+request.path()); try { // Resolves the HTTP method and fails if the method is invalid requestMethod = request.method(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index b1337172a5679..05c273c3acfa4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -23,12 +23,14 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicNameValuePair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -118,8 +120,19 @@ private HttpEntity createEntity(List> bodies, Map parameters = headerValue.v2(); + if(parameters.containsKey(MediaType.COMPATIBLE_WITH_PARAMETER_NAME)){ + contentType = contentType.withParameters(new BasicNameValuePair(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, + String.valueOf(Version.CURRENT.minimumRestCompatibilityVersion().major))); + } + + return new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length, contentType); } else { XContentType xContentType = getContentType(headers, STREAMING_CONTENT_TYPES); List bytesRefList = new ArrayList<>(bodies.size()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index 7243b8b8ad49e..11f1e9ef3d4f0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -58,7 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } responseMediaType = sqlMediaTypeParser.getMediaType(request, sqlRequest); + if(responseMediaType == null){ + } long startNanos = System.nanoTime(); return channel -> client.execute(SqlQueryAction.INSTANCE, sqlRequest, new RestResponseListener(channel) { @Override @@ -66,7 +68,7 @@ public RestResponse buildResponse(SqlQueryResponse response) throws Exception { RestResponse restResponse; // XContent branch - if (responseMediaType != null && responseMediaType instanceof XContentType) { + if (responseMediaType instanceof XContentType) { XContentType type = (XContentType) responseMediaType; XContentBuilder builder = channel.newBuilder(request.getXContentType(), type, true); response.toXContent(builder, request); From ca9aa4d28d1728151ee5f6d75ad8abb24188619d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 17 Nov 2020 16:30:11 +0100 Subject: [PATCH 03/29] checkstyle' --- .../org/elasticsearch/common/xcontent/XContentBuilder.java | 3 ++- .../java/org/elasticsearch/rest/AbstractRestChannel.java | 3 ++- .../main/java/org/elasticsearch/rest/RestController.java | 7 +------ .../xpack/enrich/action/EnrichShardMultiSearchAction.java | 3 ++- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 154544d909e2a..a95574e18565c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -198,7 +198,8 @@ public XContentBuilder(XContentType xContentType, OutputStream bos, Set * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. * @param responseContentTypeString f */ - public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, String responseContentTypeString) throws IOException { + public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, + String responseContentTypeString) throws IOException { this.bos = os; this.responseContentTypeString = responseContentTypeString; this.generator = xContent.createGenerator(bos, includes, excludes); diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 4755725f55c53..cd1400f545b46 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -135,7 +135,8 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); XContentBuilder builder = - new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes, responseContentTypeString); + new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, + includes, excludes, responseContentTypeString); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index cbf4957db53b9..513ac9e582bc6 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.path.PathTrie; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.internal.io.Streams; @@ -45,8 +44,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -330,12 +327,10 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel final String uri = request.uri(); final RestRequest.Method requestMethod; - Collection x = new ArrayList(); - Iterator iterator = x.iterator(); Version compatibleVersion = this.compatibleVersion. get(request.getParsedAccept(), request.getParsedContentType(), request.hasContent()); - System.out.println(request.getParsedContentType()!=null ? request.getParsedContentType().getParameters().get(MediaType.COMPATIBLE_WITH_PARAMETER_NAME) : "null "+request.path()); + try { // Resolves the HTTP method and fails if the method is invalid requestMethod = request.method(); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 22ffe831ea5f0..d57ae067b1bfc 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -296,7 +296,8 @@ private static BytesReference filterSource(FetchSourceContext fetchSourceContext new BytesStreamOutput(source.length()), includes, excludes, - XContentType.SMILE.mediaType()); + XContentType.SMILE.mediaType() + ); XContentParser sourceParser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, From d396dbf5b196d01349b1024902a2ddb0ea2dc4ed Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 17 Nov 2020 17:20:36 +0100 Subject: [PATCH 04/29] line wrap --- .../test/rest/yaml/ClientYamlTestExecutionContext.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 05c273c3acfa4..fa4975e24f472 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -122,7 +122,8 @@ private HttpEntity createEntity(List> bodies, Map Date: Fri, 20 Nov 2020 10:29:34 +0100 Subject: [PATCH 05/29] guess what was the response type --- .../common/xcontent/ParsedMediaType.java | 13 ++++++++++--- .../common/xcontent/XContentBuilder.java | 1 + .../org/elasticsearch/rest/AbstractRestChannel.java | 10 ++++++++++ .../java/org/elasticsearch/rest/RestRequest.java | 4 ++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java index 2323c582f829f..1ef346de2873f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -33,18 +33,21 @@ * @see MediaTypeRegistry */ public class ParsedMediaType { + private final String originalHeaderValue; private final String type; private final String subType; private final Map parameters; // tchar pattern as defined by RFC7230 section 3.2.6 private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); - private ParsedMediaType(String type, String subType, Map parameters) { + private ParsedMediaType(String originalHeaderValue, String type, String subType, Map parameters) { + this.originalHeaderValue = originalHeaderValue; this.type = type; this.subType = subType; this.parameters = Collections.unmodifiableMap(parameters); } + /** * The parsed mime type without the associated parameters. Will always return lowercase. */ @@ -79,7 +82,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { throw new IllegalArgumentException("invalid media-type [" + headerValue + "]"); } if (elements.length == 1) { - return new ParsedMediaType(splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); + return new ParsedMediaType(headerValue, splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); } else { Map parameters = new HashMap<>(); for (int i = 1; i < elements.length; i++) { @@ -96,7 +99,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { String parameterValue = keyValueParam[1].toLowerCase(Locale.ROOT).trim(); parameters.put(parameterName, parameterValue); } - return new ParsedMediaType(splitMediaType[0].trim().toLowerCase(Locale.ROOT), + return new ParsedMediaType(headerValue, splitMediaType[0].trim().toLowerCase(Locale.ROOT), splitMediaType[1].trim().toLowerCase(Locale.ROOT), parameters); } } @@ -144,4 +147,8 @@ private boolean isValidParameter(String paramName, String value, Map includes, String responseContentTypeString) throws IOException { this.bos = os; this.responseContentTypeString = responseContentTypeString; + Objects.requireNonNull(responseContentTypeString); this.generator = xContent.createGenerator(bos, includes, excludes); } diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index cd1400f545b46..bb22c4cafda5e 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -121,9 +121,19 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } else { // default to JSON output when all else fails responseContentType = XContentType.JSON; + } responseContentTypeString = responseContentType.mediaType(); } + if (responseContentTypeString == null) { + if (Strings.hasText(format)) { + responseContentTypeString = XContentType.fromFormat(format).mediaType(); + } else if( request.getParsedAccept()!=null){ + responseContentTypeString = request.getParsedAccept().getOriginalHeaderValue(); + } else { + responseContentTypeString = XContentType.JSON.mediaType(); + } + } Set includes = Collections.emptySet(); Set excludes = Collections.emptySet(); diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index aedf3f026367a..669f71478bace 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -196,6 +196,10 @@ public static RestRequest requestWithoutParameters(NamedXContentRegistry xConten requestIdGenerator.incrementAndGet()); } + public String getResponseContentType(XContentType requestContentType) { + return null; + } + public enum Method { GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT } From 51ade0842f0d1f26edb04f472be12bbe5bcd2548 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 20 Nov 2020 11:12:04 +0100 Subject: [PATCH 06/29] take media type from provided xconttenttype --- .../java/org/elasticsearch/rest/AbstractRestChannel.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index bb22c4cafda5e..84f97c77b88e6 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -125,7 +125,10 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } responseContentTypeString = responseContentType.mediaType(); } - if (responseContentTypeString == null) { + if (responseContentTypeString == null && responseContentType!=null) { + responseContentTypeString = responseContentType.mediaType(); + } + if (responseContentTypeString == null ) { if (Strings.hasText(format)) { responseContentTypeString = XContentType.fromFormat(format).mediaType(); } else if( request.getParsedAccept()!=null){ From a8315bec7a55e6010722b39dcb5e1360d311ff08 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 20 Nov 2020 17:24:14 +0100 Subject: [PATCH 07/29] remove redudnant ifs --- .../rest/AbstractRestChannel.java | 16 ++----- .../org/elasticsearch/rest/RestRequest.java | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 84f97c77b88e6..606d531140386 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -101,6 +101,10 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boo public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nullable XContentType responseContentType, boolean useFiltering) throws IOException { String responseContentTypeString = null; + if (responseContentType!=null) { + responseContentTypeString = responseContentType.mediaType(); + } + if (responseContentType == null) { if (Strings.hasText(format)) { responseContentType = XContentType.fromFormat(format); @@ -125,18 +129,6 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } responseContentTypeString = responseContentType.mediaType(); } - if (responseContentTypeString == null && responseContentType!=null) { - responseContentTypeString = responseContentType.mediaType(); - } - if (responseContentTypeString == null ) { - if (Strings.hasText(format)) { - responseContentTypeString = XContentType.fromFormat(format).mediaType(); - } else if( request.getParsedAccept()!=null){ - responseContentTypeString = request.getParsedAccept().getOriginalHeaderValue(); - } else { - responseContentTypeString = XContentType.JSON.mediaType(); - } - } Set includes = Collections.emptySet(); Set excludes = Collections.emptySet(); diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index 669f71478bace..e1c1b5cd055f9 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -31,6 +31,8 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.ToXContent; @@ -71,11 +73,13 @@ public class RestRequest implements ToXContent.Params { private final HttpChannel httpChannel; private final ParsedMediaType parsedAccept; private final ParsedMediaType parsedContentType; + private XContentType responseContentType; private HttpRequest httpRequest; private boolean contentConsumed = false; private final long requestId; + private String responseContentTypeString; public boolean isContentConsumed() { return contentConsumed; @@ -108,8 +112,42 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map this.rawPath = path; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; + } +// Tuple responseContentType(MediaTypeRegistry) responseMediaTypeRegistry) { +// String responseContentTypeString = null; +// if (responseContentType!=null) { +// responseContentTypeString = responseContentType.mediaType(); +// } +// +// if (Strings.hasText(header("format"))) { +// this.responseContentType = XContentType.fromFormat(header("format")); +// this.responseContentTypeString = responseContentType.mediaType(); +// } else if(parsedAccept != null) { +// parsedAccept.toMediaType() +// } +// if (responseContentType == null && Strings.hasText(acceptHeader)) { +// responseContentType = XContentType.fromMediaType(acceptHeader); +// responseContentTypeString = acceptHeader; +// } +// +// // try to determine the response content type from the media type or the format query string parameter, with the format parameter +// // taking precedence over the Accept header +// if (responseContentType == null) { +// if (requestContentType != null) { +// // if there was a parsed content-type for the incoming request use that since no format was specified using the query +// // string parameter or the HTTP Accept header +// responseContentType = requestContentType; +// } else { +// // default to JSON output when all else fails +// responseContentType = XContentType.JSON; +// +// } +// responseContentTypeString = responseContentType.mediaType(); +// } +// } + private static @Nullable ParsedMediaType parseHeaderWithMediaType(Map> headers, String headerName) { //TODO: make all usages of headers case-insensitive List header = headers.get(headerName); @@ -532,6 +570,13 @@ public ParsedMediaType getParsedContentType() { return parsedContentType; } + public XContentType getResponseContentType() { + return responseContentType; + } + + public String getResponseContentTypeString() { + return responseContentTypeString; + } /** * Parses the given content type string for the media type. This method currently ignores parameters. */ From c65c5eff80bcf0a897d465994c5427aa5e82ce8e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 24 Nov 2020 08:49:02 +0100 Subject: [PATCH 08/29] removed unused code --- .../common/xcontent/ParsedMediaType.java | 13 ++--- .../common/xcontent/XContentBuilder.java | 1 - .../rest/AbstractRestChannel.java | 1 - .../elasticsearch/rest/RestController.java | 3 +- .../org/elasticsearch/rest/RestRequest.java | 49 ------------------- .../yaml/ClientYamlTestExecutionContext.java | 18 +------ .../xpack/sql/plugin/RestSqlQueryAction.java | 4 +- 7 files changed, 7 insertions(+), 82 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java index 1ef346de2873f..2323c582f829f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -33,21 +33,18 @@ * @see MediaTypeRegistry */ public class ParsedMediaType { - private final String originalHeaderValue; private final String type; private final String subType; private final Map parameters; // tchar pattern as defined by RFC7230 section 3.2.6 private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+"); - private ParsedMediaType(String originalHeaderValue, String type, String subType, Map parameters) { - this.originalHeaderValue = originalHeaderValue; + private ParsedMediaType(String type, String subType, Map parameters) { this.type = type; this.subType = subType; this.parameters = Collections.unmodifiableMap(parameters); } - /** * The parsed mime type without the associated parameters. Will always return lowercase. */ @@ -82,7 +79,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { throw new IllegalArgumentException("invalid media-type [" + headerValue + "]"); } if (elements.length == 1) { - return new ParsedMediaType(headerValue, splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); + return new ParsedMediaType(splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); } else { Map parameters = new HashMap<>(); for (int i = 1; i < elements.length; i++) { @@ -99,7 +96,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { String parameterValue = keyValueParam[1].toLowerCase(Locale.ROOT).trim(); parameters.put(parameterName, parameterValue); } - return new ParsedMediaType(headerValue, splitMediaType[0].trim().toLowerCase(Locale.ROOT), + return new ParsedMediaType(splitMediaType[0].trim().toLowerCase(Locale.ROOT), splitMediaType[1].trim().toLowerCase(Locale.ROOT), parameters); } } @@ -147,8 +144,4 @@ private boolean isValidParameter(String paramName, String value, Map includes, String responseContentTypeString) throws IOException { this.bos = os; this.responseContentTypeString = responseContentTypeString; - Objects.requireNonNull(responseContentTypeString); this.generator = xContent.createGenerator(bos, includes, excludes); } diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 606d531140386..25464b5bd1fb0 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -125,7 +125,6 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } else { // default to JSON output when all else fails responseContentType = XContentType.JSON; - } responseContentTypeString = responseContentType.mediaType(); } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 513ac9e582bc6..10386d19a2030 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -175,6 +175,7 @@ private void registerHandlerNoWrap(RestRequest.Method method, String path, RestH final Version version = maybeWrappedHandler.compatibleWithVersion(); assert Version.CURRENT.minimumRestCompatibilityVersion() == version || Version.CURRENT == version : "REST API compatibility is only supported for version " + Version.CURRENT.minimumRestCompatibilityVersion().major; + handlers.insertOrUpdate(path, new MethodHandlers(path, maybeWrappedHandler, method), (mHandlers, newMHandler) -> mHandlers.addMethods(maybeWrappedHandler, method)); } @@ -327,10 +328,8 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel final String uri = request.uri(); final RestRequest.Method requestMethod; - Version compatibleVersion = this.compatibleVersion. get(request.getParsedAccept(), request.getParsedContentType(), request.hasContent()); - try { // Resolves the HTTP method and fails if the method is invalid requestMethod = request.method(); diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index e1c1b5cd055f9..aedf3f026367a 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -31,8 +31,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.MediaType; -import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.ToXContent; @@ -73,13 +71,11 @@ public class RestRequest implements ToXContent.Params { private final HttpChannel httpChannel; private final ParsedMediaType parsedAccept; private final ParsedMediaType parsedContentType; - private XContentType responseContentType; private HttpRequest httpRequest; private boolean contentConsumed = false; private final long requestId; - private String responseContentTypeString; public boolean isContentConsumed() { return contentConsumed; @@ -112,42 +108,8 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map this.rawPath = path; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; - } -// Tuple responseContentType(MediaTypeRegistry) responseMediaTypeRegistry) { -// String responseContentTypeString = null; -// if (responseContentType!=null) { -// responseContentTypeString = responseContentType.mediaType(); -// } -// -// if (Strings.hasText(header("format"))) { -// this.responseContentType = XContentType.fromFormat(header("format")); -// this.responseContentTypeString = responseContentType.mediaType(); -// } else if(parsedAccept != null) { -// parsedAccept.toMediaType() -// } -// if (responseContentType == null && Strings.hasText(acceptHeader)) { -// responseContentType = XContentType.fromMediaType(acceptHeader); -// responseContentTypeString = acceptHeader; -// } -// -// // try to determine the response content type from the media type or the format query string parameter, with the format parameter -// // taking precedence over the Accept header -// if (responseContentType == null) { -// if (requestContentType != null) { -// // if there was a parsed content-type for the incoming request use that since no format was specified using the query -// // string parameter or the HTTP Accept header -// responseContentType = requestContentType; -// } else { -// // default to JSON output when all else fails -// responseContentType = XContentType.JSON; -// -// } -// responseContentTypeString = responseContentType.mediaType(); -// } -// } - private static @Nullable ParsedMediaType parseHeaderWithMediaType(Map> headers, String headerName) { //TODO: make all usages of headers case-insensitive List header = headers.get(headerName); @@ -234,10 +196,6 @@ public static RestRequest requestWithoutParameters(NamedXContentRegistry xConten requestIdGenerator.incrementAndGet()); } - public String getResponseContentType(XContentType requestContentType) { - return null; - } - public enum Method { GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT } @@ -570,13 +528,6 @@ public ParsedMediaType getParsedContentType() { return parsedContentType; } - public XContentType getResponseContentType() { - return responseContentType; - } - - public String getResponseContentTypeString() { - return responseContentTypeString; - } /** * Parses the given content type string for the media type. This method currently ignores parameters. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index fa4975e24f472..b1337172a5679 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -23,14 +23,12 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicNameValuePair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -120,20 +118,8 @@ private HttpEntity createEntity(List> bodies, Map parameters = headerValue.v2(); - if(parameters.containsKey(MediaType.COMPATIBLE_WITH_PARAMETER_NAME)){ - contentType = contentType.withParameters(new BasicNameValuePair(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, - String.valueOf(Version.CURRENT.minimumRestCompatibilityVersion().major))); - } - - return new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length, contentType); + return new ByteArrayEntity(bytesRef.bytes, bytesRef.offset, bytesRef.length, + ContentType.create(xContentType.mediaTypeWithoutParameters(), StandardCharsets.UTF_8)); } else { XContentType xContentType = getContentType(headers, STREAMING_CONTENT_TYPES); List bytesRefList = new ArrayList<>(bodies.size()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java index 11f1e9ef3d4f0..7243b8b8ad49e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java @@ -58,9 +58,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } responseMediaType = sqlMediaTypeParser.getMediaType(request, sqlRequest); - if(responseMediaType == null){ - } long startNanos = System.nanoTime(); return channel -> client.execute(SqlQueryAction.INSTANCE, sqlRequest, new RestResponseListener(channel) { @Override @@ -68,7 +66,7 @@ public RestResponse buildResponse(SqlQueryResponse response) throws Exception { RestResponse restResponse; // XContent branch - if (responseMediaType instanceof XContentType) { + if (responseMediaType != null && responseMediaType instanceof XContentType) { XContentType type = (XContentType) responseMediaType; XContentBuilder builder = channel.newBuilder(request.getXContentType(), type, true); response.toXContent(builder, request); From 48db8766deced54b3dd708c773854dd46cb0367b Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 25 Nov 2020 15:39:59 +0100 Subject: [PATCH 09/29] format response content type basing on params --- .../common/xcontent/XContentType.java | 129 ++++++++++++++++-- .../rest/AbstractRestChannel.java | 9 +- 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index c1cd11aef5f19..953afd69398d2 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -61,7 +62,106 @@ public Set headerValues() { return Set.of( new HeaderValue("application/json"), new HeaderValue("application/x-ndjson"), - new HeaderValue("application/*"), + new HeaderValue("application/*")); + } + }, + /** + * The jackson based smile binary format. Fast and compact binary format. + */ + SMILE(1) { + @Override + public String mediaTypeWithoutParameters() { + return "application/smile"; + } + + @Override + public String queryParameter() { + return "smile"; + } + + @Override + public XContent xContent() { + return SmileXContent.smileXContent; + } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/smile")); + } + }, + /** + * A YAML based content type. + */ + YAML(2) { + @Override + public String mediaTypeWithoutParameters() { + return "application/yaml"; + } + + @Override + public String queryParameter() { + return "yaml"; + } + + @Override + public XContent xContent() { + return YamlXContent.yamlXContent; + } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/yaml")); + } + }, + /** + * A CBOR based content type. + */ + CBOR(3) { + @Override + public String mediaTypeWithoutParameters() { + return "application/cbor"; + } + + @Override + public String queryParameter() { + return "cbor"; + } + + @Override + public XContent xContent() { + return CborXContent.cborXContent; + } + + @Override + public Set headerValues() { + return Set.of( + new HeaderValue("application/cbor")); + } + }, + /** + * A JSON based content type. + */ + VND_JSON(4) { + @Override + public String mediaTypeWithoutParameters() { + return VENDOR_APPLICATION_PREFIX + "json"; + } + + @Override + public String queryParameter() { + return "json"; + } + + @Override + public XContent xContent() { + return JsonXContent.jsonXContent; + } + + @Override + public Set headerValues() { + return Set.of( new HeaderValue(VENDOR_APPLICATION_PREFIX + "json", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)), new HeaderValue(VENDOR_APPLICATION_PREFIX + "x-ndjson", @@ -71,10 +171,10 @@ public Set headerValues() { /** * The jackson based smile binary format. Fast and compact binary format. */ - SMILE(1) { + VND_SMILE(5) { @Override public String mediaTypeWithoutParameters() { - return "application/smile"; + return VENDOR_APPLICATION_PREFIX + "smile"; } @Override @@ -90,7 +190,6 @@ public XContent xContent() { @Override public Set headerValues() { return Set.of( - new HeaderValue("application/smile"), new HeaderValue(VENDOR_APPLICATION_PREFIX + "smile", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } @@ -98,10 +197,10 @@ public Set headerValues() { /** * A YAML based content type. */ - YAML(2) { + VND_YAML(6) { @Override public String mediaTypeWithoutParameters() { - return "application/yaml"; + return VENDOR_APPLICATION_PREFIX + "yaml"; } @Override @@ -117,7 +216,6 @@ public XContent xContent() { @Override public Set headerValues() { return Set.of( - new HeaderValue("application/yaml"), new HeaderValue(VENDOR_APPLICATION_PREFIX + "yaml", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } @@ -125,10 +223,10 @@ public Set headerValues() { /** * A CBOR based content type. */ - CBOR(3) { + VND_CBOR(7) { @Override public String mediaTypeWithoutParameters() { - return "application/cbor"; + return VENDOR_APPLICATION_PREFIX + "cbor"; } @Override @@ -144,12 +242,10 @@ public XContent xContent() { @Override public Set headerValues() { return Set.of( - new HeaderValue("application/cbor"), new HeaderValue(VENDOR_APPLICATION_PREFIX + "cbor", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } }; - public static final MediaTypeRegistry MEDIA_TYPE_REGISTRY = new MediaTypeRegistry() .register(XContentType.values()); public static final String VENDOR_APPLICATION_PREFIX = "application/vnd.elasticsearch+"; @@ -207,4 +303,15 @@ public String mediaType() { public abstract XContent xContent(); public abstract String mediaTypeWithoutParameters(); + + public String responseContentTypeHeader(Map parameters) { + return this.mediaTypeWithoutParameters() + formatParameters(parameters); + } + + private String formatParameters(Map parameters) { + String joined = parameters.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(";")); + return joined.isEmpty() ? "" : ";" + joined; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 25464b5bd1fb0..cade071343dc5 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -100,19 +100,13 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boo @Override public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nullable XContentType responseContentType, boolean useFiltering) throws IOException { - String responseContentTypeString = null; - if (responseContentType!=null) { - responseContentTypeString = responseContentType.mediaType(); - } if (responseContentType == null) { if (Strings.hasText(format)) { responseContentType = XContentType.fromFormat(format); - responseContentTypeString = responseContentType.mediaType(); } if (responseContentType == null && Strings.hasText(acceptHeader)) { responseContentType = XContentType.fromMediaType(acceptHeader); - responseContentTypeString = acceptHeader; } } // try to determine the response content type from the media type or the format query string parameter, with the format parameter @@ -126,7 +120,6 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu // default to JSON output when all else fails responseContentType = XContentType.JSON; } - responseContentTypeString = responseContentType.mediaType(); } Set includes = Collections.emptySet(); @@ -140,7 +133,7 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, - includes, excludes, responseContentTypeString); + includes, excludes, responseContentType.responseContentTypeHeader(request.getParsedAccept().getParameters())); if (pretty) { builder.prettyPrint().lfAtEnd(); } From 227e0f8d7943deabd55428ffe810f1a2e2dea247 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 1 Dec 2020 14:57:43 +0100 Subject: [PATCH 10/29] xcontent factory for vnd --- .../elasticsearch/common/xcontent/XContentFactory.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index 38bc251be41dd..02e968f440b8d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -121,6 +121,14 @@ public static XContentBuilder contentBuilder(XContentType type) throws IOExcepti return YamlXContent.contentBuilder(); } else if (type == XContentType.CBOR) { return CborXContent.contentBuilder(); + } else if (type == XContentType.VND_JSON) { + return JsonXContent.contentBuilder(); + } else if (type == XContentType.VND_SMILE) { + return SmileXContent.contentBuilder(); + } else if (type == XContentType.VND_YAML) { + return YamlXContent.contentBuilder(); + } else if (type == XContentType.VND_CBOR) { + return CborXContent.contentBuilder(); } throw new IllegalArgumentException("No matching content type for " + type); } From d94b4dc73a2f5e656e750ec7c79ef801762c70cf Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 2 Dec 2020 16:29:48 +0100 Subject: [PATCH 11/29] text fixtures --- .../common/xcontent/XContentFactory.java | 24 ++---- .../xcontent/MapXContentParserTests.java | 5 +- .../common/xcontent/ParsedMediaTypeTests.java | 22 +++++- .../common/xcontent/XContentParserTests.java | 4 + .../rest/AbstractRestChannel.java | 4 +- x | 74 +++++++++++++++++++ 6 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 x diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index 02e968f440b8d..d6d27e746aa58 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -97,13 +97,13 @@ public static XContentBuilder cborBuilder(OutputStream os) throws IOException { * Constructs a xcontent builder that will output the result into the provided output stream. */ public static XContentBuilder contentBuilder(XContentType type, OutputStream outputStream) throws IOException { - if (type == XContentType.JSON) { + if (type == XContentType.JSON || type == XContentType.VND_JSON ) { return jsonBuilder(outputStream); - } else if (type == XContentType.SMILE) { + } else if (type == XContentType.SMILE || type == XContentType.VND_SMILE) { return smileBuilder(outputStream); - } else if (type == XContentType.YAML) { + } else if (type == XContentType.YAML || type == XContentType.VND_YAML) { return yamlBuilder(outputStream); - } else if (type == XContentType.CBOR) { + } else if (type == XContentType.CBOR || type == XContentType.VND_CBOR) { return cborBuilder(outputStream); } throw new IllegalArgumentException("No matching content type for " + type); @@ -113,21 +113,13 @@ public static XContentBuilder contentBuilder(XContentType type, OutputStream out * Returns a binary content builder for the provided content type. */ public static XContentBuilder contentBuilder(XContentType type) throws IOException { - if (type == XContentType.JSON) { + if (type == XContentType.JSON || type == XContentType.VND_JSON) { return JsonXContent.contentBuilder(); - } else if (type == XContentType.SMILE) { + } else if (type == XContentType.SMILE || type == XContentType.VND_SMILE) { return SmileXContent.contentBuilder(); - } else if (type == XContentType.YAML) { + } else if (type == XContentType.YAML || type == XContentType.VND_YAML) { return YamlXContent.contentBuilder(); - } else if (type == XContentType.CBOR) { - return CborXContent.contentBuilder(); - } else if (type == XContentType.VND_JSON) { - return JsonXContent.contentBuilder(); - } else if (type == XContentType.VND_SMILE) { - return SmileXContent.contentBuilder(); - } else if (type == XContentType.VND_YAML) { - return YamlXContent.contentBuilder(); - } else if (type == XContentType.VND_CBOR) { + } else if (type == XContentType.CBOR || type == XContentType.VND_CBOR) { return CborXContent.contentBuilder(); } throw new IllegalArgumentException("No matching content type for " + type); diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java index b3fb9eee4662d..0644ab2974578 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java @@ -91,7 +91,7 @@ public void compareTokens(CheckedConsumer consumer try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { try (XContentParser mapParser = new MapXContentParser( xContentRegistry(), LoggingDeprecationHandler.INSTANCE, map, xContentType)) { - assertEquals(parser.contentType(), mapParser.contentType()); +// assertEquals(parser.contentType(), mapParser.contentType()); XContentParser.Token token; assertEquals(parser.currentToken(), mapParser.currentToken()); assertEquals(parser.currentName(), mapParser.currentName()); @@ -101,7 +101,8 @@ public void compareTokens(CheckedConsumer consumer assertEquals(token, mapToken); assertEquals(parser.currentName(), mapParser.currentName()); if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) { - if (xContentType != XContentType.YAML || token != XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + if ((xContentType != XContentType.YAML && xContentType != XContentType.VND_YAML) || + token != XContentParser.Token.VALUE_EMBEDDED_OBJECT) { // YAML struggles with converting byte arrays into text, because it // does weird base64 decoding to the values. We don't do this // weirdness in the MapXContentParser, so don't try to stringify it. diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java index 73bc8e2e5734f..d548c3694ac6e 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ParsedMediaTypeTests.java @@ -33,6 +33,26 @@ public class ParsedMediaTypeTests extends ESTestCase { MediaTypeRegistry mediaTypeRegistry = new MediaTypeRegistry() .register(XContentType.values()); + public void testCanonicalParsing() { + assertThat(ParsedMediaType.parseMediaType("application/json") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.JSON)); + assertThat(ParsedMediaType.parseMediaType("application/yaml") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.YAML)); + assertThat(ParsedMediaType.parseMediaType("application/smile") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.SMILE)); + assertThat(ParsedMediaType.parseMediaType("application/cbor") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.CBOR)); + + assertThat(ParsedMediaType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=7") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_JSON)); + assertThat(ParsedMediaType.parseMediaType("application/vnd.elasticsearch+yaml;compatible-with=7") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_YAML)); + assertThat(ParsedMediaType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=7") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_SMILE)); + assertThat(ParsedMediaType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=7") + .toMediaType(mediaTypeRegistry), equalTo(XContentType.VND_CBOR)); + } + public void testJsonWithParameters() throws Exception { String mediaType = "application/vnd.elasticsearch+json"; assertThat(ParsedMediaType.parseMediaType(mediaType).getParameters(), @@ -48,7 +68,7 @@ public void testJsonWithParameters() throws Exception { public void testWhiteSpaceInTypeSubtype() { String mediaType = " application/vnd.elasticsearch+json "; assertThat(ParsedMediaType.parseMediaType(mediaType).toMediaType(mediaTypeRegistry), - equalTo(XContentType.JSON)); + equalTo(XContentType.VND_JSON)); assertThat(ParsedMediaType.parseMediaType(mediaType + "; compatible-with=123; charset=UTF-8").getParameters(), equalTo(Map.of("charset", "utf-8", "compatible-with", "123"))); diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java index 262b25b97b26e..31b4e381c7ae5 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java @@ -79,10 +79,14 @@ public void testFloat() throws IOException { assertEquals(value, number.floatValue(), 0.0f); switch (xContentType) { + case VND_CBOR: + case VND_SMILE: case CBOR: case SMILE: assertThat(number, instanceOf(Float.class)); break; + case VND_JSON: + case VND_YAML: case JSON: case YAML: assertThat(number, instanceOf(Double.class)); diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index cade071343dc5..74e29b0666730 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.Collections; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; @@ -131,9 +132,10 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); + Map parameters = request.getParsedAccept() != null ? request.getParsedAccept().getParameters() : Collections.emptyMap(); XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, - includes, excludes, responseContentType.responseContentTypeHeader(request.getParsedAccept().getParameters())); + includes, excludes, responseContentType.responseContentTypeHeader(parameters)); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/x b/x new file mode 100644 index 0000000000000..e95f4e8ba2f9c --- /dev/null +++ b/x @@ -0,0 +1,74 @@ +> Task :buildSrc:reaper:compileJava UP-TO-DATE +> Task :buildSrc:reaper:processResources NO-SOURCE +> Task :buildSrc:reaper:classes UP-TO-DATE +> Task :buildSrc:reaper:jar UP-TO-DATE +> Task :buildSrc:reaper:assemble UP-TO-DATE +> Task :buildSrc:reaper:compileTestJava NO-SOURCE +> Task :buildSrc:reaper:processTestResources NO-SOURCE +> Task :buildSrc:reaper:testClasses UP-TO-DATE +> Task :buildSrc:reaper:test NO-SOURCE +> Task :buildSrc:reaper:check UP-TO-DATE +> Task :buildSrc:reaper:build UP-TO-DATE +> Task :buildSrc:compileJava UP-TO-DATE +> Task :buildSrc:compileGroovy UP-TO-DATE +> Task :buildSrc:generateVersionProperties UP-TO-DATE +> Task :buildSrc:pluginDescriptors UP-TO-DATE +> Task :buildSrc:processResources UP-TO-DATE +> Task :buildSrc:classes UP-TO-DATE +> Task :buildSrc:jar UP-TO-DATE +> Task :buildSrc:assemble UP-TO-DATE +> Task :buildSrc:pluginUnderTestMetadata UP-TO-DATE +> Task :buildSrc:compileTestFixturesJava UP-TO-DATE +> Task :buildSrc:compileTestFixturesGroovy NO-SOURCE +> Task :buildSrc:processTestFixturesResources NO-SOURCE +> Task :buildSrc:testFixturesClasses UP-TO-DATE +> Task :buildSrc:testFixturesJar UP-TO-DATE +> Task :buildSrc:compileTestJava UP-TO-DATE +> Task :buildSrc:compileTestGroovy NO-SOURCE +> Task :buildSrc:processTestResources UP-TO-DATE +> Task :buildSrc:testClasses UP-TO-DATE +> Task :buildSrc:test SKIPPED +> Task :buildSrc:validatePlugins UP-TO-DATE +> Task :buildSrc:check UP-TO-DATE +> Task :buildSrc:build UP-TO-DATE + +> Configure project : +========================= WARNING ========================= + Backwards compatibility tests are disabled! +See https://github.com/elastic/elasticsearch/pull/65601 +=========================================================== +======================================= +Elasticsearch Build Hamster says Hello! + Gradle Version : 6.6.1 + OS Info : Mac OS X 10.14.6 (x86_64) + JDK Version : 14 (OpenJDK) + JAVA_HOME : /Library/Java/JavaVirtualMachines/jdk-14.0.2.jdk/Contents/Home + Random Testing Seed : 9947573E0259FE77 + In FIPS 140 mode : false +======================================= + +> Task :dependencies + +------------------------------------------------------------ +Root project - Elasticsearch subproject : +------------------------------------------------------------ + +es_distro_extracted_local +No dependencies + +es_distro_extracted_testclusters--runTask-0-8.0.0-SNAPSHOT +No dependencies + +es_distro_file_local +No dependencies + +es_distro_file_testclusters--runTask-0-8.0.0-SNAPSHOT +No dependencies + +jdk_provisioned_runtime +\--- adoptopenjdk_15:mac:15.0.1 + +A web-based, searchable dependency report is available by adding the --scan option. + +BUILD SUCCESSFUL in 3s +1 actionable task: 1 executed From 91fcf3a3c59692e64372f57a1c4cea8269402066 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 2 Dec 2020 16:44:27 +0100 Subject: [PATCH 12/29] precommit --- .../main/java/org/elasticsearch/rest/AbstractRestChannel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 74e29b0666730..4799707be1242 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -132,7 +132,8 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } OutputStream unclosableOutputStream = Streams.flushOnCloseStream(bytesOutput()); - Map parameters = request.getParsedAccept() != null ? request.getParsedAccept().getParameters() : Collections.emptyMap(); + Map parameters = request.getParsedAccept() != null ? + request.getParsedAccept().getParameters() : Collections.emptyMap(); XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, includes, excludes, responseContentType.responseContentTypeHeader(parameters)); From a085dc6c33a355db06b99f541e41958f1168aec4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 3 Dec 2020 15:36:48 +0100 Subject: [PATCH 13/29] test fixes --- .../elasticsearch/common/xcontent/XContentType.java | 10 +++++----- .../painless/action/PainlessExecuteRequestTests.java | 2 +- .../common/xcontent/XContentParserUtilsTests.java | 2 +- .../common/xcontent/XContentTypeTests.java | 12 ++++++------ .../ingest/PipelineConfigurationTests.java | 2 +- .../rest/action/cat/RestTableTests.java | 6 ++++-- .../action/document/RestGetSourceActionTests.java | 2 +- .../search/searchafter/SearchAfterBuilderTests.java | 2 +- .../xpack/watcher/common/text/TextTemplate.java | 3 ++- 9 files changed, 22 insertions(+), 19 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 953afd69398d2..7824287eab482 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -151,8 +151,8 @@ public String mediaTypeWithoutParameters() { @Override public String queryParameter() { - return "json"; - } + return "vnd_json"; + } // another hack.. @Override public XContent xContent() { @@ -179,7 +179,7 @@ public String mediaTypeWithoutParameters() { @Override public String queryParameter() { - return "smile"; + return "vnd_smile"; } @Override @@ -205,7 +205,7 @@ public String mediaTypeWithoutParameters() { @Override public String queryParameter() { - return "yaml"; + return "vnd_yaml"; } @Override @@ -231,7 +231,7 @@ public String mediaTypeWithoutParameters() { @Override public String queryParameter() { - return "cbor"; + return "vnd_cbor"; } @Override diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java index 79d37cc0e5bc7..5f369e0d853fc 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java @@ -48,7 +48,7 @@ public class PainlessExecuteRequestTests extends AbstractWireSerializingTestCase // Testing XContent serialization manually here, because the xContentType field in ContextSetup determines // how the request needs to parse and the xcontent serialization framework randomizes that. The XContentType - // is not known and accessable when the test request instance is created in the xcontent serialization framework. + // is not known and accessible when the test request instance is created in the xcontent serialization framework. // Changing that is a big change. Writing a custom xcontent test here is the best option for now, because as far // as I know this request class is the only case where this is a problem. public final void testFromXContent() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java index ad34f5b9a0987..92c1e55cee2fc 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java @@ -91,7 +91,7 @@ public void testStoredFieldsValueBoolean() throws IOException { public void testStoredFieldsValueBinary() throws IOException { final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8"); assertParseFieldsSimpleValue(value, (xcontentType, result) -> { - if (xcontentType == XContentType.JSON) { + if (xcontentType == XContentType.JSON || xcontentType == XContentType.VND_JSON) { // binary values will be parsed back and returned as base64 strings when reading from json assertArrayEquals(value, Base64.getDecoder().decode((String) result)); } else { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java index 72bdf549b8d40..8345e57da04a5 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java @@ -99,21 +99,21 @@ public void testFromRubbish() throws Exception { public void testVersionedMediaType() { String version = String.valueOf(randomNonNegativeByte()); assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+json;compatible-with=" + version), - equalTo(XContentType.JSON)); + equalTo(XContentType.VND_JSON)); assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+cbor;compatible-with=" + version), - equalTo(XContentType.CBOR)); + equalTo(XContentType.VND_CBOR)); assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+smile;compatible-with=" + version), - equalTo(XContentType.SMILE)); + equalTo(XContentType.VND_SMILE)); assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+yaml;compatible-with=" + version), - equalTo(XContentType.YAML)); + equalTo(XContentType.VND_YAML)); assertThat(XContentType.fromMediaType("application/json"), equalTo(XContentType.JSON)); assertThat(XContentType.fromMediaType("application/vnd.elasticsearch+x-ndjson;compatible-with=" + version), - equalTo(XContentType.JSON)); + equalTo(XContentType.VND_JSON)); assertThat(XContentType.fromMediaType("APPLICATION/VND.ELASTICSEARCH+JSON;COMPATIBLE-WITH=" + version), - equalTo(XContentType.JSON)); + equalTo(XContentType.VND_JSON)); assertThat(XContentType.fromMediaType("APPLICATION/JSON"), equalTo(XContentType.JSON)); } diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java index eb1171f66a597..25fa45660002e 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -65,7 +65,7 @@ public void testParser() throws IOException { XContentParser xContentParser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput()); PipelineConfiguration parsed = parser.parse(xContentParser, null); - assertEquals(xContentType, parsed.getXContentType()); +// assertEquals(xContentType, parsed.getXContentType()); assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); assertEquals("1", parsed.getId()); } diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java index 4f7bc1bedd9d6..78aeb6269f468 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java @@ -37,6 +37,7 @@ import static org.elasticsearch.rest.action.cat.RestTable.buildResponse; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.equalToIgnoringCase; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; @@ -269,8 +270,9 @@ private RestResponse assertResponseContentType(Map> headers public void sendResponse(RestResponse response) { } }); - - assertThat(response.contentType(), equalTo(mediaType)); + String actualWithoutWhitespaces = mediaType.replaceAll("\\s+",""); + String expectedWithoutWhitespaces = response.contentType().replaceAll("\\s+",""); + assertThat(expectedWithoutWhitespaces, equalToIgnoringCase(actualWithoutWhitespaces)); return response; } diff --git a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java index 38a3fdf1af617..c69103531d48d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/document/RestGetSourceActionTests.java @@ -63,7 +63,7 @@ public void testRestGetSourceAction() throws Exception { final RestResponse restResponse = listener.buildResponse(response); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json"));//dropping charset as it was not on a request assertThat(restResponse.content(), equalTo(new BytesArray("{\"foo\": \"bar\"}"))); } diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index 45da49e5f4c48..cafe5778ef52d 100644 --- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -200,7 +200,7 @@ public void testFromXContentIllegalType() throws Exception { for (XContentType type : XContentType.values()) { // BIG_DECIMAL // ignore json and yaml, they parse floating point numbers as floats/doubles - if (type == XContentType.JSON || type == XContentType.YAML) { + if (type == XContentType.JSON || type == XContentType.YAML || type == XContentType.VND_JSON || type == XContentType.VND_YAML) { continue; } XContentBuilder xContent = XContentFactory.contentBuilder(type); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java index b082ab57f4410..f6c55f9967a86 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java @@ -43,7 +43,8 @@ public TextTemplate(String template, @Nullable XContentType contentType, ScriptT if (type == ScriptType.INLINE) { options = new HashMap<>(); if (contentType != null) { - options.put(Script.CONTENT_TYPE_OPTION, contentType.mediaType()); + // vnd_json->xcontent->json->application/json - ugly hack :( + options.put(Script.CONTENT_TYPE_OPTION, contentType.xContent().type().mediaType()); } } if (params == null) { From 7aeabf7a09505778be0fe134d758e501e0627fe2 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 3 Dec 2020 21:14:08 +0100 Subject: [PATCH 14/29] fix serialisation --- .../percolator/PercolateQueryBuilder.java | 4 ++-- .../http/DetailedErrorsDisabledIT.java | 2 +- .../org/elasticsearch/http/NoHandlerIT.java | 2 +- .../storedscripts/PutStoredScriptRequest.java | 4 ++-- .../action/index/IndexRequest.java | 4 ++-- .../action/ingest/PutPipelineRequest.java | 5 ++-- .../ingest/SimulatePipelineRequest.java | 5 ++-- .../termvectors/TermVectorsRequest.java | 4 ++-- .../common/xcontent/XContentHelper.java | 24 +++++++++++++++++++ .../index/query/MoreLikeThisQueryBuilder.java | 6 +++-- .../ingest/PipelineConfiguration.java | 4 ++-- .../xpack/watcher/PutWatchRequest.java | 5 ++-- .../xpack/core/enrich/EnrichPolicy.java | 4 ++-- .../xpack/core/ml/action/PostDataAction.java | 5 ++-- .../monitoring/action/MonitoringBulkDoc.java | 5 ++-- .../support/xcontent/XContentSource.java | 5 ++-- .../actions/execute/ExecuteWatchRequest.java | 5 ++-- .../logstash/action/PutPipelineRequest.java | 5 ++-- 18 files changed, 66 insertions(+), 32 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index e9e73278c5e8a..b2f571fb8c04a 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -226,7 +226,7 @@ protected PercolateQueryBuilder(String field, Supplier documentS } documents = in.readList(StreamInput::readBytesReference); if (documents.isEmpty() == false) { - documentXContentType = in.readEnum(XContentType.class); + documentXContentType = XContentHelper.readFromWire(in); } else { documentXContentType = null; } @@ -272,7 +272,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeBytesReference(document); } if (documents.isEmpty() == false) { - out.writeEnum(documentXContentType); + XContentHelper.writeTo(out, documentXContentType); } } diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java index 17a1f3b5d256c..3b67e7440cec3 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java @@ -54,7 +54,7 @@ public void testThatErrorTraceParamReturns400() throws IOException { getRestClient().performRequest(request)); Response response = e.getResponse(); - assertThat(response.getHeader("Content-Type"), is("application/json; charset=UTF-8")); + assertThat(response.getHeader("Content-Type"), is("application/json")); assertThat(EntityUtils.toString(e.getResponse().getEntity()), containsString("\"error\":\"error traces in responses are disabled.\"")); assertThat(response.getStatusLine().getStatusCode(), is(400)); diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/NoHandlerIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/NoHandlerIT.java index d3707031f0e35..62b4747ae2512 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/NoHandlerIT.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/NoHandlerIT.java @@ -36,7 +36,7 @@ public class NoHandlerIT extends HttpSmokeTestCase { public void testNoHandlerRespectsAcceptHeader() throws IOException { runTestNoHandlerRespectsAcceptHeader( "application/json", - "application/json; charset=UTF-8", + "application/json", "\"error\":\"no handler found for uri [/foo/bar/baz/qux/quux] and method [GET]\""); runTestNoHandlerRespectsAcceptHeader( "application/yaml", diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 85ec9b14f097e..a78a6fbf30b4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -47,7 +47,7 @@ public PutStoredScriptRequest(StreamInput in) throws IOException { super(in); id = in.readOptionalString(); content = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); context = in.readOptionalString(); source = new StoredScriptSource(in); } @@ -127,7 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeOptionalString(id); out.writeBytesReference(content); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); out.writeOptionalString(context); source.writeTo(out); } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 01c9ca49a9c43..7f208aa3158a1 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -146,7 +146,7 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.readBoolean()) { - contentType = in.readEnum(XContentType.class); + contentType = XContentHelper.readFromWire(in); } else { contentType = null; } @@ -657,7 +657,7 @@ private void writeBody(StreamOutput out) throws IOException { out.writeLong(autoGeneratedTimestamp); if (contentType != null) { out.writeBoolean(true); - out.writeEnum(contentType); + XContentHelper.writeTo(out, contentType); } else { out.writeBoolean(false); } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index 212921f0e5a95..b0dea42cf94cd 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -50,7 +51,7 @@ public PutPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); } PutPipelineRequest() { @@ -78,7 +79,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeBytesReference(source); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 414d8835f4faa..27990857f7b38 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.ConfigurationUtils; @@ -63,7 +64,7 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); } @Override @@ -101,7 +102,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBoolean(verbose); out.writeBytesReference(source); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index b35b3f7bca472..bb6e24ac5393d 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -143,7 +143,7 @@ public TermVectorsRequest() { if (in.readBoolean()) { doc = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); } routing = in.readOptionalString(); preference = in.readOptionalString(); @@ -493,7 +493,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeBytesReference(doc); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } out.writeOptionalString(routing); out.writeOptionalString(preference); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 5b828e06f1009..e0cecd08065db 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -20,12 +20,14 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.Compressor; import org.elasticsearch.common.compress.CompressorFactory; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent.Params; import java.io.BufferedInputStream; @@ -453,4 +455,26 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio builder.copyCurrentStructure(parser); return BytesReference.bytes(builder); } + + public static XContentType readFromWire(org.elasticsearch.common.io.stream.StreamInput in) throws IOException { +// int ordinal = in.readVInt();// have to be reading int, value can be either 0-4 or 0-7 +// +// +// XContentType[] values = XContentType.values(); +// if (ordinal < 0 || ordinal >= values.length) { +// throw new IOException("Unknown " + XContentType.class.getSimpleName() + " ordinal [" + ordinal + "]"); +// } +// return values[ordinal]; + return in.readEnum(XContentType.class); + + + } + + public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException { + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeVInt(xContentType.ordinal() % 4); + } else { + out.writeVInt(xContentType.ordinal()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index 44d51ee82c0cf..61e1540537c44 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -219,7 +220,7 @@ public Item(@Nullable String index, XContentBuilder doc) { } if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); } else { id = in.readString(); } @@ -240,7 +241,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(doc != null); if (doc != null) { out.writeGenericValue(doc); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); + } else { out.writeString(id); } diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index 81ef55ecf40a6..e96640dd700b2 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -117,7 +117,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); + return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentHelper.readFromWire(in)); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -133,7 +133,7 @@ public String toString() { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(config); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index fbdea0cf04067..c8f4ff7a9f9ad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -46,7 +47,7 @@ public PutWatchRequest(StreamInput in) throws IOException { id = in.readString(); source = in.readBytesReference(); active = in.readBoolean(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in);; version = in.readZLong(); ifSeqNo = in.readZLong(); ifPrimaryTerm = in.readVLong(); @@ -64,7 +65,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeBytesReference(source); out.writeBoolean(active); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); out.writeZLong(version); out.writeZLong(ifSeqNo); out.writeVLong(ifPrimaryTerm); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index a6a8e90f8fe97..a2726465463ed 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -235,7 +235,7 @@ public static class QuerySource implements Writeable { private final XContentType contentType; QuerySource(StreamInput in) throws IOException { - this(in.readBytesReference(), in.readEnum(XContentType.class)); + this(in.readBytesReference(), XContentHelper.readFromWire(in)); } public QuerySource(BytesReference query, XContentType contentType) { @@ -258,7 +258,7 @@ public XContentType getContentType() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeBytesReference(query); - out.writeEnum(contentType); + XContentHelper.writeTo(out, contentType); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java index b0d69a87dc075..6743768b092a2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; @@ -111,7 +112,7 @@ public Request(StreamInput in) throws IOException { dataDescription = in.readOptionalWriteable(DataDescription::new); content = in.readBytesReference(); if (in.readBoolean()) { - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in);; } } @@ -125,7 +126,7 @@ public void writeTo(StreamOutput out) throws IOException { boolean hasXContentType = xContentType != null; out.writeBoolean(hasXContentType); if (hasXContentType) { - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java index c95dbb03a1d46..8683c9f548235 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.monitoring.MonitoredSystem; @@ -53,7 +54,7 @@ public MonitoringBulkDoc (StreamInput in) throws IOException { this.type = in.readOptionalString(); this.id = in.readOptionalString(); this.source = in.readBytesReference(); - this.xContentType = (source != BytesArray.EMPTY) ? in.readEnum(XContentType.class) : XContentType.JSON; + this.xContentType = (source != BytesArray.EMPTY) ? XContentHelper.readFromWire(in) : XContentType.JSON; this.intervalMillis = in.readVLong(); } @@ -65,7 +66,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeBytesReference(source); if (source != BytesArray.EMPTY) { - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } out.writeVLong(intervalMillis); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java index 55362390216a3..89a7685a0acff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentUtils; @@ -120,12 +121,12 @@ public XContentParser parser(NamedXContentRegistry xContentRegistry, InputStream } public static XContentSource readFrom(StreamInput in) throws IOException { - return new XContentSource(in.readBytesReference(), in.readEnum(XContentType.class)); + return new XContentSource(in.readBytesReference(), XContentHelper.readFromWire(in)); } public static void writeTo(XContentSource source, StreamOutput out) throws IOException { out.writeBytesReference(source.bytes); - out.writeEnum(source.contentType); + XContentHelper.writeTo(out, source.contentType); } private Object data() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java index d45a1a5c8b2b4..863e40f45e59e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.watcher.client.WatchSourceBuilder; import org.elasticsearch.xpack.core.watcher.execution.ActionExecutionMode; @@ -69,7 +70,7 @@ public ExecuteWatchRequest(StreamInput in) throws IOException { } if (in.readBoolean()) { watchSource = in.readBytesReference(); - xContentType = in.readEnum(XContentType.class); + xContentType = XContentHelper.readFromWire(in); } debug = in.readBoolean(); } @@ -96,7 +97,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(watchSource != null); if (watchSource != null) { out.writeBytesReference(watchSource); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } out.writeBoolean(debug); } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java index 525be7f2a1cff..1cdbe843bef91 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; @@ -31,7 +32,7 @@ public PutPipelineRequest(StreamInput in) throws IOException { super(in); this.id = in.readString(); this.source = in.readString(); - this.xContentType = in.readEnum(XContentType.class); + this.xContentType = XContentHelper.readFromWire(in); } public String id() { @@ -51,7 +52,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(id); out.writeString(source); - out.writeEnum(xContentType); + XContentHelper.writeTo(out, xContentType); } @Override From d59449538248b66676e4fe501e0ab0f0d905cc20 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 7 Dec 2020 14:56:07 +0100 Subject: [PATCH 15/29] fixing restcontroller --- .../main/java/org/elasticsearch/client/RequestConverters.java | 3 ++- .../java/org/elasticsearch/client/RequestConvertersTests.java | 3 ++- .../src/main/java/org/elasticsearch/rest/RestController.java | 3 ++- .../org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index d5ddbc2c17592..b970a609334dc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -1177,7 +1177,8 @@ Params withWaitForEvents(Priority waitForEvents) { */ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) { XContentType requestContentType = indexRequest.getContentType(); - if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE) { + if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE && + requestContentType != XContentType.VND_JSON && requestContentType != XContentType.VND_SMILE) { throw new IllegalArgumentException("Unsupported content-type found for request with content-type [" + requestContentType + "], only JSON and SMILE are supported"); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index c0b6b93c4544f..57c9fc43d4edb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1821,7 +1821,8 @@ public void testCreateContentType() { } public void testEnforceSameContentType() { - XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE); + //todo this is not finished.. + XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.VND_JSON, XContentType.VND_SMILE); IndexRequest indexRequest = new IndexRequest().source(singletonMap("field", "value"), xContentType); assertEquals(xContentType, enforceSameContentType(indexRequest, null)); assertEquals(xContentType, enforceSameContentType(indexRequest, xContentType)); diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 7a232127f650e..7da53d72a2174 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -237,7 +237,8 @@ private void dispatchRequest(RestRequest request, RestChannel channel, RestHandl sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel); return; } - if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.SMILE) { + if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.VND_JSON + && xContentType != XContentType.SMILE && xContentType != XContentType.VND_SMILE) { channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_ACCEPTABLE, "Content-Type [" + xContentType + "] does not support stream parsing. Use JSON or SMILE instead")); return; diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index c17c0c8086f67..d8692650b68cd 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -288,7 +288,8 @@ private void assertQuery(String sql, String columnName, String columnType, Objec private Map runSql(Mode mode, String sql, boolean columnar) throws IOException { Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT); String requestContent = query(sql).mode(mode).toString(); - String format = randomFrom(XContentType.values()).name().toLowerCase(Locale.ROOT); + String format = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.CBOR, XContentType.YAML) + .name().toLowerCase(Locale.ROOT); // add a client_id to the request if (randomBoolean()) { From 8c520ce91822c59324463063b6a131208588c794 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 8 Dec 2020 13:36:22 +0100 Subject: [PATCH 16/29] spotless --- .../org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java index d8692650b68cd..a7e6e637c3543 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/SqlProtocolTestCase.java @@ -288,8 +288,8 @@ private void assertQuery(String sql, String columnName, String columnType, Objec private Map runSql(Mode mode, String sql, boolean columnar) throws IOException { Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT); String requestContent = query(sql).mode(mode).toString(); - String format = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.CBOR, XContentType.YAML) - .name().toLowerCase(Locale.ROOT); + String format = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.CBOR, XContentType.YAML).name() + .toLowerCase(Locale.ROOT); // add a client_id to the request if (randomBoolean()) { From 521b5ad5e580ddedd30be5f948bfbc33a9ec0480 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 8 Dec 2020 14:35:09 +0100 Subject: [PATCH 17/29] small cleanup --- .../client/RequestConverters.java | 5 +- .../common/xcontent/XContentFactory.java | 19 +++-- .../common/xcontent/XContentType.java | 24 ++++++ .../xcontent/MapXContentParserTests.java | 2 +- .../common/xcontent/XContentHelper.java | 10 --- .../index/query/MoreLikeThisQueryBuilder.java | 1 - .../elasticsearch/rest/RestController.java | 5 +- .../xcontent/XContentParserUtilsTests.java | 2 +- .../ingest/PipelineConfigurationTests.java | 2 +- .../searchafter/SearchAfterBuilderTests.java | 2 +- x | 74 ------------------- .../watcher/common/text/TextTemplate.java | 3 +- 12 files changed, 45 insertions(+), 104 deletions(-) delete mode 100644 x diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index b970a609334dc..620be506444db 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -1177,15 +1177,14 @@ Params withWaitForEvents(Priority waitForEvents) { */ static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) { XContentType requestContentType = indexRequest.getContentType(); - if (requestContentType != XContentType.JSON && requestContentType != XContentType.SMILE && - requestContentType != XContentType.VND_JSON && requestContentType != XContentType.VND_SMILE) { + if (requestContentType.canonical() != XContentType.JSON && requestContentType.canonical() != XContentType.SMILE) { throw new IllegalArgumentException("Unsupported content-type found for request with content-type [" + requestContentType + "], only JSON and SMILE are supported"); } if (xContentType == null) { return requestContentType; } - if (requestContentType != xContentType) { + if (requestContentType.canonical() != xContentType.canonical()) { throw new IllegalArgumentException("Mismatching content-type found for request with content-type [" + requestContentType + "], previous requests have content-type [" + xContentType + "]"); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java index d6d27e746aa58..d0a611dc02836 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentFactory.java @@ -97,13 +97,14 @@ public static XContentBuilder cborBuilder(OutputStream os) throws IOException { * Constructs a xcontent builder that will output the result into the provided output stream. */ public static XContentBuilder contentBuilder(XContentType type, OutputStream outputStream) throws IOException { - if (type == XContentType.JSON || type == XContentType.VND_JSON ) { + XContentType canonical = type.canonical(); + if (canonical == XContentType.JSON) { return jsonBuilder(outputStream); - } else if (type == XContentType.SMILE || type == XContentType.VND_SMILE) { + } else if (canonical == XContentType.SMILE) { return smileBuilder(outputStream); - } else if (type == XContentType.YAML || type == XContentType.VND_YAML) { + } else if (canonical == XContentType.YAML) { return yamlBuilder(outputStream); - } else if (type == XContentType.CBOR || type == XContentType.VND_CBOR) { + } else if (canonical == XContentType.CBOR) { return cborBuilder(outputStream); } throw new IllegalArgumentException("No matching content type for " + type); @@ -113,13 +114,15 @@ public static XContentBuilder contentBuilder(XContentType type, OutputStream out * Returns a binary content builder for the provided content type. */ public static XContentBuilder contentBuilder(XContentType type) throws IOException { - if (type == XContentType.JSON || type == XContentType.VND_JSON) { + XContentType canonical = type.canonical(); + + if (canonical == XContentType.JSON) { return JsonXContent.contentBuilder(); - } else if (type == XContentType.SMILE || type == XContentType.VND_SMILE) { + } else if (canonical == XContentType.SMILE) { return SmileXContent.contentBuilder(); - } else if (type == XContentType.YAML || type == XContentType.VND_YAML) { + } else if (canonical == XContentType.YAML) { return YamlXContent.contentBuilder(); - } else if (type == XContentType.CBOR || type == XContentType.VND_CBOR) { + } else if (canonical == XContentType.CBOR) { return CborXContent.contentBuilder(); } throw new IllegalArgumentException("No matching content type for " + type); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 7824287eab482..15b352ee5a11b 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -167,6 +167,11 @@ public Set headerValues() { new HeaderValue(VENDOR_APPLICATION_PREFIX + "x-ndjson", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } + + @Override + public XContentType canonical() { + return JSON; + } }, /** * The jackson based smile binary format. Fast and compact binary format. @@ -193,6 +198,11 @@ public Set headerValues() { new HeaderValue(VENDOR_APPLICATION_PREFIX + "smile", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } + + @Override + public XContentType canonical() { + return SMILE; + } }, /** * A YAML based content type. @@ -219,6 +229,11 @@ public Set headerValues() { new HeaderValue(VENDOR_APPLICATION_PREFIX + "yaml", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } + + @Override + public XContentType canonical() { + return YAML; + } }, /** * A CBOR based content type. @@ -245,6 +260,11 @@ public Set headerValues() { new HeaderValue(VENDOR_APPLICATION_PREFIX + "cbor", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } + + @Override + public XContentType canonical() { + return CBOR; + } }; public static final MediaTypeRegistry MEDIA_TYPE_REGISTRY = new MediaTypeRegistry() .register(XContentType.values()); @@ -314,4 +334,8 @@ private String formatParameters(Map parameters) { .collect(Collectors.joining(";")); return joined.isEmpty() ? "" : ";" + joined; } + + public XContentType canonical(){ + return this; + } } diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java index 0644ab2974578..80bf80f873f6c 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java @@ -101,7 +101,7 @@ public void compareTokens(CheckedConsumer consumer assertEquals(token, mapToken); assertEquals(parser.currentName(), mapParser.currentName()); if (token != null && (token.isValue() || token == XContentParser.Token.VALUE_NULL)) { - if ((xContentType != XContentType.YAML && xContentType != XContentType.VND_YAML) || + if ((xContentType.canonical() != XContentType.YAML) || token != XContentParser.Token.VALUE_EMBEDDED_OBJECT) { // YAML struggles with converting byte arrays into text, because it // does weird base64 decoding to the values. We don't do this diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index e0cecd08065db..a326b5c71046e 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -457,17 +457,7 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio } public static XContentType readFromWire(org.elasticsearch.common.io.stream.StreamInput in) throws IOException { -// int ordinal = in.readVInt();// have to be reading int, value can be either 0-4 or 0-7 -// -// -// XContentType[] values = XContentType.values(); -// if (ordinal < 0 || ordinal >= values.length) { -// throw new IOException("Unknown " + XContentType.class.getSimpleName() + " ordinal [" + ordinal + "]"); -// } -// return values[ordinal]; return in.readEnum(XContentType.class); - - } public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index 61e1540537c44..571a09e6f21a9 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -242,7 +242,6 @@ public void writeTo(StreamOutput out) throws IOException { if (doc != null) { out.writeGenericValue(doc); XContentHelper.writeTo(out, xContentType); - } else { out.writeString(id); } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 7da53d72a2174..e3db4a46fb13b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -237,8 +237,9 @@ private void dispatchRequest(RestRequest request, RestChannel channel, RestHandl sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel); return; } - if (handler.supportsContentStream() && xContentType != XContentType.JSON && xContentType != XContentType.VND_JSON - && xContentType != XContentType.SMILE && xContentType != XContentType.VND_SMILE) { + //todo consider moving to XContentType + if (handler.supportsContentStream() && xContentType.canonical() != XContentType.JSON + && xContentType.canonical() != XContentType.SMILE) { channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_ACCEPTABLE, "Content-Type [" + xContentType + "] does not support stream parsing. Use JSON or SMILE instead")); return; diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java index 92c1e55cee2fc..af52963b50d9a 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java @@ -91,7 +91,7 @@ public void testStoredFieldsValueBoolean() throws IOException { public void testStoredFieldsValueBinary() throws IOException { final byte[] value = randomUnicodeOfLength(scaledRandomIntBetween(10, 1000)).getBytes("UTF-8"); assertParseFieldsSimpleValue(value, (xcontentType, result) -> { - if (xcontentType == XContentType.JSON || xcontentType == XContentType.VND_JSON) { + if (xcontentType.canonical() == XContentType.JSON) { // binary values will be parsed back and returned as base64 strings when reading from json assertArrayEquals(value, Base64.getDecoder().decode((String) result)); } else { diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java index 25fa45660002e..fdc0bf4b5b652 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -65,7 +65,7 @@ public void testParser() throws IOException { XContentParser xContentParser = xContentType.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, bytes.streamInput()); PipelineConfiguration parsed = parser.parse(xContentParser, null); -// assertEquals(xContentType, parsed.getXContentType()); + assertEquals(xContentType.canonical(), parsed.getXContentType()); assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); assertEquals("1", parsed.getId()); } diff --git a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index cafe5778ef52d..e6038a13adac0 100644 --- a/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -200,7 +200,7 @@ public void testFromXContentIllegalType() throws Exception { for (XContentType type : XContentType.values()) { // BIG_DECIMAL // ignore json and yaml, they parse floating point numbers as floats/doubles - if (type == XContentType.JSON || type == XContentType.YAML || type == XContentType.VND_JSON || type == XContentType.VND_YAML) { + if (type.canonical() == XContentType.JSON || type.canonical() == XContentType.YAML) { continue; } XContentBuilder xContent = XContentFactory.contentBuilder(type); diff --git a/x b/x deleted file mode 100644 index e95f4e8ba2f9c..0000000000000 --- a/x +++ /dev/null @@ -1,74 +0,0 @@ -> Task :buildSrc:reaper:compileJava UP-TO-DATE -> Task :buildSrc:reaper:processResources NO-SOURCE -> Task :buildSrc:reaper:classes UP-TO-DATE -> Task :buildSrc:reaper:jar UP-TO-DATE -> Task :buildSrc:reaper:assemble UP-TO-DATE -> Task :buildSrc:reaper:compileTestJava NO-SOURCE -> Task :buildSrc:reaper:processTestResources NO-SOURCE -> Task :buildSrc:reaper:testClasses UP-TO-DATE -> Task :buildSrc:reaper:test NO-SOURCE -> Task :buildSrc:reaper:check UP-TO-DATE -> Task :buildSrc:reaper:build UP-TO-DATE -> Task :buildSrc:compileJava UP-TO-DATE -> Task :buildSrc:compileGroovy UP-TO-DATE -> Task :buildSrc:generateVersionProperties UP-TO-DATE -> Task :buildSrc:pluginDescriptors UP-TO-DATE -> Task :buildSrc:processResources UP-TO-DATE -> Task :buildSrc:classes UP-TO-DATE -> Task :buildSrc:jar UP-TO-DATE -> Task :buildSrc:assemble UP-TO-DATE -> Task :buildSrc:pluginUnderTestMetadata UP-TO-DATE -> Task :buildSrc:compileTestFixturesJava UP-TO-DATE -> Task :buildSrc:compileTestFixturesGroovy NO-SOURCE -> Task :buildSrc:processTestFixturesResources NO-SOURCE -> Task :buildSrc:testFixturesClasses UP-TO-DATE -> Task :buildSrc:testFixturesJar UP-TO-DATE -> Task :buildSrc:compileTestJava UP-TO-DATE -> Task :buildSrc:compileTestGroovy NO-SOURCE -> Task :buildSrc:processTestResources UP-TO-DATE -> Task :buildSrc:testClasses UP-TO-DATE -> Task :buildSrc:test SKIPPED -> Task :buildSrc:validatePlugins UP-TO-DATE -> Task :buildSrc:check UP-TO-DATE -> Task :buildSrc:build UP-TO-DATE - -> Configure project : -========================= WARNING ========================= - Backwards compatibility tests are disabled! -See https://github.com/elastic/elasticsearch/pull/65601 -=========================================================== -======================================= -Elasticsearch Build Hamster says Hello! - Gradle Version : 6.6.1 - OS Info : Mac OS X 10.14.6 (x86_64) - JDK Version : 14 (OpenJDK) - JAVA_HOME : /Library/Java/JavaVirtualMachines/jdk-14.0.2.jdk/Contents/Home - Random Testing Seed : 9947573E0259FE77 - In FIPS 140 mode : false -======================================= - -> Task :dependencies - ------------------------------------------------------------- -Root project - Elasticsearch subproject : ------------------------------------------------------------- - -es_distro_extracted_local -No dependencies - -es_distro_extracted_testclusters--runTask-0-8.0.0-SNAPSHOT -No dependencies - -es_distro_file_local -No dependencies - -es_distro_file_testclusters--runTask-0-8.0.0-SNAPSHOT -No dependencies - -jdk_provisioned_runtime -\--- adoptopenjdk_15:mac:15.0.1 - -A web-based, searchable dependency report is available by adding the --scan option. - -BUILD SUCCESSFUL in 3s -1 actionable task: 1 executed diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java index f6c55f9967a86..45bdf149ebae4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/text/TextTemplate.java @@ -43,8 +43,7 @@ public TextTemplate(String template, @Nullable XContentType contentType, ScriptT if (type == ScriptType.INLINE) { options = new HashMap<>(); if (contentType != null) { - // vnd_json->xcontent->json->application/json - ugly hack :( - options.put(Script.CONTENT_TYPE_OPTION, contentType.xContent().type().mediaType()); + options.put(Script.CONTENT_TYPE_OPTION, contentType.canonical().mediaType()); } } if (params == null) { From 261df305596e234ec96acdb929e4dfed091af20e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 8 Dec 2020 16:46:41 +0100 Subject: [PATCH 18/29] tests relying on xcontent being in cannonical form - not chaning --- .../painless/action/PainlessExecuteRequestTests.java | 2 +- .../org/elasticsearch/ingest/IngestServiceTests.java | 2 +- .../java/org/elasticsearch/rest/RestControllerTests.java | 9 +++------ .../java/org/elasticsearch/search/SearchHitTests.java | 7 ++++--- .../java/org/elasticsearch/search/SearchHitsTests.java | 8 ++++---- .../org/elasticsearch/test/AbstractWireTestCase.java | 3 ++- .../org/elasticsearch/test/AbstractXContentTestCase.java | 2 +- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java index 5f369e0d853fc..698c52a55dc33 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteRequestTests.java @@ -107,7 +107,7 @@ private static ContextSetup randomContextSetup() { String index = randomBoolean() ? randomAlphaOfLength(4) : null; QueryBuilder query = randomBoolean() ? new MatchAllQueryBuilder() : null; BytesReference doc = null; - XContentType xContentType = randomFrom(XContentType.values()); + XContentType xContentType = randomFrom(XContentType.values()).canonical(); if (randomBoolean()) { try { XContentBuilder xContentBuilder = XContentBuilder.builder(xContentType.xContent()); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 56f88769fbe12..b0dd16187089e 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -1020,7 +1020,7 @@ public void testBulkRequestExecution() throws Exception { for (DocWriteRequest docWriteRequest : bulkRequest.requests()) { IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(docWriteRequest); assertThat(indexRequest, notNullValue()); - assertThat(indexRequest.getContentType(), equalTo(xContentType)); + assertThat(indexRequest.getContentType(), equalTo(xContentType.canonical())); } } diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 91f229e7b60aa..7c4e5c93e787e 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -62,7 +63,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -730,11 +730,8 @@ public Version compatibleWithVersion() { } private String randomCompatibleMimeType(byte version) { - String subtype = randomFrom(Stream.of(XContentType.values()) - .map(XContentType::mediaTypeWithoutParameters) - .toArray(String[]::new)) - .split("/")[1]; - return randomFrom("application/vnd.elasticsearch+" + subtype + ";compatible-with=" + version); + return randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML) + .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version))); } private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index 0a6bef86af27a..a75c615c64144 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -62,8 +62,9 @@ import static org.hamcrest.Matchers.nullValue; public class SearchHitTests extends AbstractWireSerializingTestCase { + public static SearchHit createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { - return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); + return createTestItem(randomFrom(XContentType.values()).canonical(), withOptionalInnerHits, withShardTarget); } public static SearchHit createTestItem(XContentType xContentType, boolean withOptionalInnerHits, boolean transportSerialization) { @@ -151,11 +152,11 @@ protected Writeable.Reader instanceReader() { @Override protected SearchHit createTestInstance() { - return createTestItem(randomFrom(XContentType.values()), randomBoolean(), randomBoolean()); + return createTestItem(randomFrom(XContentType.values()).canonical(), randomBoolean(), randomBoolean()); } public void testFromXContent() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); + XContentType xContentType = randomFrom(XContentType.values()).canonical(); SearchHit searchHit = createTestItem(xContentType, true, false); boolean humanReadable = randomBoolean(); BytesReference originalBytes = toShuffledXContent(searchHit, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java index 86147270d218b..2fc488145ce4f 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitsTests.java @@ -44,7 +44,7 @@ public class SearchHitsTests extends AbstractSerializingTestCase { public static SearchHits createTestItem(boolean withOptionalInnerHits, boolean withShardTarget) { - return createTestItem(randomFrom(XContentType.values()), withOptionalInnerHits, withShardTarget); + return createTestItem(randomFrom(XContentType.values()).canonical(), withOptionalInnerHits, withShardTarget); } private static SearchHit[] createSearchHitArray(int size, XContentType xContentType, boolean withOptionalInnerHits, @@ -105,7 +105,7 @@ protected SearchHits mutateInstance(SearchHits instance) { switch (randomIntBetween(0, 5)) { case 0: return new SearchHits(createSearchHitArray(instance.getHits().length + 1, - randomFrom(XContentType.values()), false, randomBoolean()), + randomFrom(XContentType.values()).canonical(), false, randomBoolean()), instance.getTotalHits(), instance.getMaxScore()); case 1: final TotalHits totalHits; @@ -176,7 +176,7 @@ protected SearchHits createTestInstance() { // This instance is used to test the transport serialization so it's fine // to produce shard targets (withShardTarget is true) since they are serialized // in this layer. - return createTestItem(randomFrom(XContentType.values()), true, true); + return createTestItem(randomFrom(XContentType.values()).canonical(), true, true); } @Override @@ -240,7 +240,7 @@ public void testFromXContentWithShards() throws IOException { long totalHits = 1000; float maxScore = 1.5f; SearchHits searchHits = new SearchHits(hits, new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), maxScore); - XContentType xContentType = randomFrom(XContentType.values()); + XContentType xContentType = randomFrom(XContentType.values()).canonical(); BytesReference bytes = toShuffledXContent(searchHits, xContentType, ToXContent.EMPTY_PARAMS, false); try (XContentParser parser = xContentType.xContent() .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, bytes.streamInput())) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java index 3997db194e9b5..92318b7c90fd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.hamcrest.Matchers; import java.io.IOException; import java.util.Collections; @@ -94,7 +95,7 @@ protected final void assertSerialization(T testInstance, Version version) throws */ protected void assertEqualInstances(T expectedInstance, T newInstance) { assertNotSame(newInstance, expectedInstance); - assertEquals(expectedInstance, newInstance); + assertThat(expectedInstance, Matchers.equalTo(newInstance)); assertEquals(expectedInstance.hashCode(), newInstance.hashCode()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java index 136f51bed3c8c..82a47aca78383 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java @@ -127,7 +127,7 @@ private XContentTester( public void test() throws IOException { for (int runs = 0; runs < numberOfTestRuns; runs++) { - XContentType xContentType = randomFrom(XContentType.values()); + XContentType xContentType = randomFrom(XContentType.values()).canonical(); T testInstance = instanceSupplier.apply(xContentType); BytesReference originalXContent = toXContent.apply(testInstance, xContentType); BytesReference shuffledContent = insertRandomFieldsAndShuffle(originalXContent, xContentType, supportsUnknownFields, From 48200e2762c66f69391e27e3eb727dc11af18d72 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 9 Dec 2020 09:13:16 +0100 Subject: [PATCH 19/29] test fix --- .../org/elasticsearch/client/RequestConvertersTests.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 57c9fc43d4edb..de1c5ce1e4fcf 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1821,10 +1821,11 @@ public void testCreateContentType() { } public void testEnforceSameContentType() { - //todo this is not finished.. XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.VND_JSON, XContentType.VND_SMILE); IndexRequest indexRequest = new IndexRequest().source(singletonMap("field", "value"), xContentType); - assertEquals(xContentType, enforceSameContentType(indexRequest, null)); + // indexRequest content type is made canonical + // ( XContentBuilder(VND_JSON).getXContentType -> JSON //hardcoded in JSonXContentGenerator ) + assertEquals(xContentType.canonical(), enforceSameContentType(indexRequest, null)); assertEquals(xContentType, enforceSameContentType(indexRequest, xContentType)); XContentType bulkContentType = randomBoolean() ? xContentType : null; @@ -1841,7 +1842,7 @@ public void testEnforceSameContentType() { assertEquals("Unsupported content-type found for request with content-type [YAML], only JSON and SMILE are supported", exception.getMessage()); - XContentType requestContentType = xContentType == XContentType.JSON ? XContentType.SMILE : XContentType.JSON; + XContentType requestContentType = xContentType.canonical() == XContentType.JSON ? XContentType.SMILE : XContentType.JSON; exception = expectThrows(IllegalArgumentException.class, () -> enforceSameContentType(new IndexRequest().source(singletonMap("field", "value"), requestContentType), xContentType)); From 5030a572289367684fb81d0680cb3408d5190820 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 10 Dec 2020 14:47:01 +0100 Subject: [PATCH 20/29] inline read and javadoc --- .../common/xcontent/XContentBuilder.java | 2 +- .../elasticsearch/common/xcontent/XContentType.java | 11 ++++++----- .../common/xcontent/MapXContentParserTests.java | 2 +- .../percolator/PercolateQueryBuilder.java | 2 +- .../cluster/storedscripts/PutStoredScriptRequest.java | 2 +- .../org/elasticsearch/action/index/IndexRequest.java | 2 +- .../action/ingest/PutPipelineRequest.java | 2 +- .../action/ingest/SimulatePipelineRequest.java | 2 +- .../action/termvectors/TermVectorsRequest.java | 2 +- .../elasticsearch/common/xcontent/XContentHelper.java | 10 ++++++---- .../index/query/MoreLikeThisQueryBuilder.java | 2 +- .../elasticsearch/ingest/PipelineConfiguration.java | 2 +- .../protocol/xpack/watcher/PutWatchRequest.java | 2 +- .../elasticsearch/xpack/core/enrich/EnrichPolicy.java | 2 +- .../xpack/core/ml/action/PostDataAction.java | 2 +- .../core/monitoring/action/MonitoringBulkDoc.java | 2 +- .../core/watcher/support/xcontent/XContentSource.java | 2 +- .../actions/execute/ExecuteWatchRequest.java | 2 +- .../xpack/logstash/action/PutPipelineRequest.java | 2 +- 19 files changed, 29 insertions(+), 26 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index a95574e18565c..f3712e6ce06a6 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -196,7 +196,7 @@ public XContentBuilder(XContentType xContentType, OutputStream bos, Set * @param os the output stream * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. - * @param responseContentTypeString f + * @param responseContentTypeString a content-type header value to be send back on a response */ public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, String responseContentTypeString) throws IOException { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 15b352ee5a11b..dc3b2110e1dc2 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -141,7 +141,7 @@ public Set headerValues() { } }, /** - * A JSON based content type. + * A versioned JSON based content type. */ VND_JSON(4) { @Override @@ -152,7 +152,7 @@ public String mediaTypeWithoutParameters() { @Override public String queryParameter() { return "vnd_json"; - } // another hack.. + } @Override public XContent xContent() { @@ -174,7 +174,7 @@ public XContentType canonical() { } }, /** - * The jackson based smile binary format. Fast and compact binary format. + * Versioned jackson based smile binary format. Fast and compact binary format. */ VND_SMILE(5) { @Override @@ -205,7 +205,7 @@ public XContentType canonical() { } }, /** - * A YAML based content type. + * A Versioned YAML based content type. */ VND_YAML(6) { @Override @@ -236,7 +236,7 @@ public XContentType canonical() { } }, /** - * A CBOR based content type. + * A Versioned CBOR based content type. */ VND_CBOR(7) { @Override @@ -266,6 +266,7 @@ public XContentType canonical() { return CBOR; } }; + public static final MediaTypeRegistry MEDIA_TYPE_REGISTRY = new MediaTypeRegistry() .register(XContentType.values()); public static final String VENDOR_APPLICATION_PREFIX = "application/vnd.elasticsearch+"; diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java index 80bf80f873f6c..21d574092456a 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java @@ -91,7 +91,7 @@ public void compareTokens(CheckedConsumer consumer try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { try (XContentParser mapParser = new MapXContentParser( xContentRegistry(), LoggingDeprecationHandler.INSTANCE, map, xContentType)) { -// assertEquals(parser.contentType(), mapParser.contentType()); + assertEquals(parser.contentType(), mapParser.contentType()); XContentParser.Token token; assertEquals(parser.currentToken(), mapParser.currentToken()); assertEquals(parser.currentName(), mapParser.currentName()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index b2f571fb8c04a..907480e5f3b9c 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -226,7 +226,7 @@ protected PercolateQueryBuilder(String field, Supplier documentS } documents = in.readList(StreamInput::readBytesReference); if (documents.isEmpty() == false) { - documentXContentType = XContentHelper.readFromWire(in); + documentXContentType = in.readEnum(XContentType.class); } else { documentXContentType = null; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index a78a6fbf30b4c..6407b8dea5c8b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -47,7 +47,7 @@ public PutStoredScriptRequest(StreamInput in) throws IOException { super(in); id = in.readOptionalString(); content = in.readBytesReference(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); context = in.readOptionalString(); source = new StoredScriptSource(in); } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 7f208aa3158a1..565b1e17fc99d 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -146,7 +146,7 @@ public IndexRequest(@Nullable ShardId shardId, StreamInput in) throws IOExceptio isRetry = in.readBoolean(); autoGeneratedTimestamp = in.readLong(); if (in.readBoolean()) { - contentType = XContentHelper.readFromWire(in); + contentType = in.readEnum(XContentType.class); } else { contentType = null; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java index b0dea42cf94cd..1007c1835f2b6 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java @@ -51,7 +51,7 @@ public PutPipelineRequest(StreamInput in) throws IOException { super(in); id = in.readString(); source = in.readBytesReference(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); } PutPipelineRequest() { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 27990857f7b38..a5818bdb0f4ec 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -64,7 +64,7 @@ public SimulatePipelineRequest(BytesReference source, XContentType xContentType) id = in.readOptionalString(); verbose = in.readBoolean(); source = in.readBytesReference(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java index bb6e24ac5393d..9ab5623870737 100644 --- a/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java @@ -143,7 +143,7 @@ public TermVectorsRequest() { if (in.readBoolean()) { doc = in.readBytesReference(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); } routing = in.readOptionalString(); preference = in.readOptionalString(); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index a326b5c71046e..55e66d7dca181 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -456,10 +456,12 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio return BytesReference.bytes(builder); } - public static XContentType readFromWire(org.elasticsearch.common.io.stream.StreamInput in) throws IOException { - return in.readEnum(XContentType.class); - } - + /** + * Serialises new XContentType vnd values in a bwc manner + * TODO remove in ES v9 + * @param out stream output of the destination node + * @param xContentType an instance to serialise + */ public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException { if (out.getVersion().before(Version.V_8_0_0)) { out.writeVInt(xContentType.ordinal() % 4); diff --git a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java index 571a09e6f21a9..6499c88587b31 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java @@ -220,7 +220,7 @@ public Item(@Nullable String index, XContentBuilder doc) { } if (in.readBoolean()) { doc = (BytesReference) in.readGenericValue(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); } else { id = in.readString(); } diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index e96640dd700b2..e0d56b4428012 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -117,7 +117,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PipelineConfiguration readFrom(StreamInput in) throws IOException { - return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentHelper.readFromWire(in)); + return new PipelineConfiguration(in.readString(), in.readBytesReference(), in.readEnum(XContentType.class)); } public static Diff readDiffFrom(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index c8f4ff7a9f9ad..897265f63565f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -47,7 +47,7 @@ public PutWatchRequest(StreamInput in) throws IOException { id = in.readString(); source = in.readBytesReference(); active = in.readBoolean(); - xContentType = XContentHelper.readFromWire(in);; + xContentType = in.readEnum(XContentType.class);; version = in.readZLong(); ifSeqNo = in.readZLong(); ifPrimaryTerm = in.readVLong(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java index a2726465463ed..396caed7a5611 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichPolicy.java @@ -235,7 +235,7 @@ public static class QuerySource implements Writeable { private final XContentType contentType; QuerySource(StreamInput in) throws IOException { - this(in.readBytesReference(), XContentHelper.readFromWire(in)); + this(in.readBytesReference(), in.readEnum(XContentType.class)); } public QuerySource(BytesReference query, XContentType contentType) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java index 6743768b092a2..bd05a1e367af4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java @@ -112,7 +112,7 @@ public Request(StreamInput in) throws IOException { dataDescription = in.readOptionalWriteable(DataDescription::new); content = in.readBytesReference(); if (in.readBoolean()) { - xContentType = XContentHelper.readFromWire(in);; + xContentType = in.readEnum(XContentType.class);; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java index 8683c9f548235..4a1218269b9a1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkDoc.java @@ -54,7 +54,7 @@ public MonitoringBulkDoc (StreamInput in) throws IOException { this.type = in.readOptionalString(); this.id = in.readOptionalString(); this.source = in.readBytesReference(); - this.xContentType = (source != BytesArray.EMPTY) ? XContentHelper.readFromWire(in) : XContentType.JSON; + this.xContentType = (source != BytesArray.EMPTY) ? in.readEnum(XContentType.class) : XContentType.JSON; this.intervalMillis = in.readVLong(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java index 89a7685a0acff..d6ec2074485ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/XContentSource.java @@ -121,7 +121,7 @@ public XContentParser parser(NamedXContentRegistry xContentRegistry, InputStream } public static XContentSource readFrom(StreamInput in) throws IOException { - return new XContentSource(in.readBytesReference(), XContentHelper.readFromWire(in)); + return new XContentSource(in.readBytesReference(), in.readEnum(XContentType.class)); } public static void writeTo(XContentSource source, StreamOutput out) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java index 863e40f45e59e..72a0f146d4cb2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/execute/ExecuteWatchRequest.java @@ -70,7 +70,7 @@ public ExecuteWatchRequest(StreamInput in) throws IOException { } if (in.readBoolean()) { watchSource = in.readBytesReference(); - xContentType = XContentHelper.readFromWire(in); + xContentType = in.readEnum(XContentType.class); } debug = in.readBoolean(); } diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java index 1cdbe843bef91..0502e9b5b3530 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/action/PutPipelineRequest.java @@ -32,7 +32,7 @@ public PutPipelineRequest(StreamInput in) throws IOException { super(in); this.id = in.readString(); this.source = in.readString(); - this.xContentType = XContentHelper.readFromWire(in); + this.xContentType = in.readEnum(XContentType.class); } public String id() { From 1ce88adeacb07035cb38a1dceb7aed6d195fd478 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 10 Dec 2020 15:45:23 +0100 Subject: [PATCH 21/29] use canonical in mapxcontentparsertests --- .../java/org/elasticsearch/client/RequestConvertersTests.java | 4 ++-- .../elasticsearch/common/xcontent/MapXContentParserTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index de1c5ce1e4fcf..247ddecff8cb8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1823,8 +1823,8 @@ public void testCreateContentType() { public void testEnforceSameContentType() { XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE, XContentType.VND_JSON, XContentType.VND_SMILE); IndexRequest indexRequest = new IndexRequest().source(singletonMap("field", "value"), xContentType); - // indexRequest content type is made canonical - // ( XContentBuilder(VND_JSON).getXContentType -> JSON //hardcoded in JSonXContentGenerator ) + // indexRequest content type is made canonical because IndexRequest's content-type is + // from XContentBuilder.getXContentType (hardcoded in JsonXContentGEnerator) assertEquals(xContentType.canonical(), enforceSameContentType(indexRequest, null)); assertEquals(xContentType, enforceSameContentType(indexRequest, xContentType)); diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java index 21d574092456a..4aaacf743015b 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/MapXContentParserTests.java @@ -91,7 +91,7 @@ public void compareTokens(CheckedConsumer consumer try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { try (XContentParser mapParser = new MapXContentParser( xContentRegistry(), LoggingDeprecationHandler.INSTANCE, map, xContentType)) { - assertEquals(parser.contentType(), mapParser.contentType()); + assertEquals(parser.contentType(), mapParser.contentType().canonical()); XContentParser.Token token; assertEquals(parser.currentToken(), mapParser.currentToken()); assertEquals(parser.currentName(), mapParser.currentName()); From f97f0c4796806af2cbe780aba39695121d42e4ae Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 16 Dec 2020 13:08:02 +0100 Subject: [PATCH 22/29] javadocs --- .../elasticsearch/common/xcontent/MediaTypeRegistry.java | 4 ++-- .../elasticsearch/common/xcontent/XContentBuilder.java | 8 ++++---- .../org/elasticsearch/common/xcontent/XContentType.java | 5 +++++ .../org/elasticsearch/common/io/stream/StreamOutput.java | 2 ++ .../org/elasticsearch/common/xcontent/XContentHelper.java | 4 ++-- .../main/java/org/elasticsearch/rest/RestController.java | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java index 8324adfa25384..674770944516d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java @@ -33,11 +33,11 @@ * I.e. txt used in path _sql?format=txt will return TextFormat.PLAIN_TEXT * * Multiple header representations may map to a single {@link MediaType} for example, "application/json" - * and "application/vnd.elasticsearch+json" both represent a JSON MediaType. + * and "application/x-ndjson" both represent a JSON MediaType. * A MediaType can have only one query parameter representation. * For example "json" (case insensitive) maps back to a JSON media type. * - * Additionally, a http header may optionally have parameters. For example "application/json; charset=utf-8". + * Additionally, a http header may optionally have parameters. For example "application/vnd.elasticsearch+json; compatible-with=7". * This class also allows to define a regular expression for valid values of charset. */ public class MediaTypeRegistry { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index f3712e6ce06a6..97c23164ab192 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -63,20 +63,20 @@ public static XContentBuilder builder(XContent xContent) throws IOException { } /** - * Create a new {@link XContentBuilder} using the given {@link XContent} content and some inclusive and/or exclusive filters. + * Create a new {@link XContentBuilder} using the given {@link XContentType} xContentType and some inclusive and/or exclusive filters. *

* The builder uses an internal {@link ByteArrayOutputStream} output stream to build the content. When both exclusive and * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the * remaining fields against the inclusive filters. *

* - * @param xContent the {@link XContent} + * @param xContentType the {@link XContentType} * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. * @throws IOException if an {@link IOException} occurs while building the content */ - public static XContentBuilder builder(XContentType xContent, Set includes, Set excludes) throws IOException { - return new XContentBuilder(xContent.xContent(), new ByteArrayOutputStream(), includes, excludes, xContent.mediaType()); + public static XContentBuilder builder(XContentType xContentType, Set includes, Set excludes) throws IOException { + return new XContentBuilder(xContentType.xContent(), new ByteArrayOutputStream(), includes, excludes, xContentType.mediaType()); } private static final Map, Writer> WRITERS; diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index dc3b2110e1dc2..a1854e1694f32 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -336,6 +336,11 @@ private String formatParameters(Map parameters) { return joined.isEmpty() ? "" : ";" + joined; } + /** + * Returns a canonical XContentType for given this XContentType. + * XContentTypes can derive from other XContentType. Like application/vnd.elasticsearch+json derives from application/json + * A canonical is the XContentType that is being derived from. + */ public XContentType canonical(){ return this; } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 9f6648de52f49..a07b66b0b7435 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.JodaCompatibleZonedDateTime; import org.joda.time.DateTimeZone; import org.joda.time.ReadableInstant; @@ -1273,6 +1274,7 @@ public void writeNamedWriteableList(List list) throws * Writes an enum with type E based on its ordinal value */ public > void writeEnum(E enumValue) throws IOException { + assert enumValue instanceof XContentType == false : "XContentHelper#writeTo should be used for XContentType serialisation"; writeVInt(enumValue.ordinal()); } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 55e66d7dca181..27ac15f672b47 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -457,10 +457,10 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio } /** - * Serialises new XContentType vnd values in a bwc manner + * Serialises new XContentType VND_ values in a bwc manner * TODO remove in ES v9 * @param out stream output of the destination node - * @param xContentType an instance to serialise + * @param xContentType an instance to serialize */ public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException { if (out.getVersion().before(Version.V_8_0_0)) { diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index e3db4a46fb13b..e0deb4d96ed2a 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -237,7 +237,7 @@ private void dispatchRequest(RestRequest request, RestChannel channel, RestHandl sendContentTypeErrorMessage(request.getAllHeaderValues("Content-Type"), channel); return; } - //todo consider moving to XContentType + //TODO consider refactoring to handler.supportsContentStream(xContentType). It is only used with JSON and SMILE if (handler.supportsContentStream() && xContentType.canonical() != XContentType.JSON && xContentType.canonical() != XContentType.SMILE) { channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_ACCEPTABLE, From 752968cc833edac449659d6219dfcb9fd5b6d019 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 17 Dec 2020 11:47:01 +0100 Subject: [PATCH 23/29] response media type to be a ParsedMediaType instead of just a string --- .../common/xcontent/ParsedMediaType.java | 25 ++++++++++++++++--- .../common/xcontent/XContentBuilder.java | 22 +++++++++------- .../common/xcontent/XContentType.java | 12 --------- .../common/xcontent/XContentHelper.java | 3 ++- .../rest/AbstractRestChannel.java | 6 ++++- .../rest/RestControllerTests.java | 4 ++- .../action/EnrichShardMultiSearchAction.java | 3 ++- .../watcher/common/http/HttpRequest.java | 4 ++- 8 files changed, 50 insertions(+), 29 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java index 504decef7a7ac..6073f91ef89f8 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -24,6 +24,7 @@ import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * A raw result of parsing media types from Accept or Content-Type headers. @@ -44,7 +45,7 @@ private ParsedMediaType(String originalHeaderValue, String type, String subType, this.originalHeaderValue = originalHeaderValue; this.type = type; this.subType = subType; - this.parameters = Collections.unmodifiableMap(parameters); + this.parameters = parameters; } /** @@ -55,7 +56,7 @@ public String mediaTypeWithoutParameters() { } public Map getParameters() { - return parameters; + return Collections.unmodifiableMap(parameters); } /** @@ -81,7 +82,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { throw new IllegalArgumentException("invalid media-type [" + headerValue + "]"); } if (elements.length == 1) { - return new ParsedMediaType(headerValue, splitMediaType[0].trim(), splitMediaType[1].trim(), Collections.emptyMap()); + return new ParsedMediaType(headerValue, splitMediaType[0].trim(), splitMediaType[1].trim(), new HashMap<>()); } else { Map parameters = new HashMap<>(); for (int i = 1; i < elements.length; i++) { @@ -105,6 +106,12 @@ public static ParsedMediaType parseMediaType(String headerValue) { return null; } + public static ParsedMediaType parseMediaType(XContentType requestContentType, Map parameters) { + ParsedMediaType parsedMediaType = parseMediaType(requestContentType.mediaType()); + parsedMediaType.parameters.putAll(parameters); + return parsedMediaType; + } + // simplistic check for media ranges. do not validate if this is a correct header private static boolean isMediaRange(String headerValue) { return headerValue.contains(","); @@ -151,4 +158,16 @@ private boolean isValidParameter(String paramName, String value, Map parameters) { + return this.mediaTypeWithoutParameters() + formatParameters(parameters); + } + + private String formatParameters(Map parameters) { + String joined = parameters.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(";")); + return joined.isEmpty() ? "" : ";" + joined; + } + } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 97c23164ab192..b19235d3e8060 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -76,7 +76,8 @@ public static XContentBuilder builder(XContent xContent) throws IOException { * @throws IOException if an {@link IOException} occurs while building the content */ public static XContentBuilder builder(XContentType xContentType, Set includes, Set excludes) throws IOException { - return new XContentBuilder(xContentType.xContent(), new ByteArrayOutputStream(), includes, excludes, xContentType.mediaType()); + return new XContentBuilder(xContentType.xContent(), new ByteArrayOutputStream(), includes, excludes, + ParsedMediaType.parseMediaType(xContentType.mediaType())); } private static final Map, Writer> WRITERS; @@ -167,14 +168,14 @@ public interface HumanReadableTransformer { private byte compatibleMajorVersion; - private String responseContentTypeString; + private ParsedMediaType responseContentType; /** * Constructs a new builder using the provided XContent and an OutputStream. Make sure * to call {@link #close()} when the builder is done with. */ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { - this(xContent, bos, Collections.emptySet(), Collections.emptySet(), xContent.type().mediaType()); + this(xContent, bos, Collections.emptySet(), Collections.emptySet(), ParsedMediaType.parseMediaType(xContent.type().mediaType())); } /** @@ -184,7 +185,7 @@ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { * {@link #close()} when the builder is done with. */ public XContentBuilder(XContentType xContentType, OutputStream bos, Set includes) throws IOException { - this(xContentType.xContent(), bos, includes, Collections.emptySet(), xContentType.mediaType()); + this(xContentType.xContent(), bos, includes, Collections.emptySet(), ParsedMediaType.parseMediaType(xContentType.mediaType())); } /** @@ -193,20 +194,23 @@ public XContentBuilder(XContentType xContentType, OutputStream bos, Set * remaining fields against the inclusive filters. *

* Make sure to call {@link #close()} when the builder is done with. - * @param os the output stream + * @param os the output stream * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. - * @param responseContentTypeString a content-type header value to be send back on a response + * @param responseContentType a content-type header value to be send back on a response */ public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, - String responseContentTypeString) throws IOException { + ParsedMediaType responseContentType) throws IOException { this.bos = os; - this.responseContentTypeString = responseContentTypeString; + assert responseContentType != null : "generated response cannot be null"; + this.responseContentType = responseContentType; this.generator = xContent.createGenerator(bos, includes, excludes); } public String getResponseContentTypeString() { - return responseContentTypeString; + Map parameters = responseContentType != null ? + responseContentType.getParameters() : Collections.emptyMap(); + return responseContentType.responseContentTypeHeader(parameters); } public XContentType contentType() { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index a1854e1694f32..baaae87ef0e2e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -325,17 +324,6 @@ public String mediaType() { public abstract String mediaTypeWithoutParameters(); - public String responseContentTypeHeader(Map parameters) { - return this.mediaTypeWithoutParameters() + formatParameters(parameters); - } - - private String formatParameters(Map parameters) { - String joined = parameters.entrySet().stream() - .map(e -> e.getKey() + "=" + e.getValue()) - .collect(Collectors.joining(";")); - return joined.isEmpty() ? "" : ";" + joined; - } - /** * Returns a canonical XContentType for given this XContentType. * XContentTypes can derive from other XContentType. Like application/vnd.elasticsearch+json derives from application/json diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 27ac15f672b47..4f4dc6d3cb51f 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -464,7 +464,8 @@ public static BytesReference childBytes(XContentParser parser) throws IOExceptio */ public static void writeTo(StreamOutput out, XContentType xContentType) throws IOException { if (out.getVersion().before(Version.V_8_0_0)) { - out.writeVInt(xContentType.ordinal() % 4); + // when sending an enumeration to parameters = request.getParsedAccept() != null ? request.getParsedAccept().getParameters() : Collections.emptyMap(); + ParsedMediaType responseMediaType = ParsedMediaType.parseMediaType(responseContentType, parameters); + XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), unclosableOutputStream, - includes, excludes, responseContentType.responseContentTypeHeader(parameters)); + includes, excludes, responseMediaType); if (pretty) { builder.prettyPrint().lfAtEnd(); } diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 7c4e5c93e787e..5683352c84d14 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.yaml.YamlXContent; @@ -730,7 +731,8 @@ public Version compatibleWithVersion() { } private String randomCompatibleMimeType(byte version) { - return randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML) + XContentType type = randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML); + return ParsedMediaType.parseMediaType(type.mediaType()) .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version))); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index d57ae067b1bfc..0709b18ecc891 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; @@ -296,7 +297,7 @@ private static BytesReference filterSource(FetchSourceContext fetchSourceContext new BytesStreamOutput(source.length()), includes, excludes, - XContentType.SMILE.mediaType() + ParsedMediaType.parseMediaType(XContentType.SMILE, emptyMap()) ); XContentParser sourceParser = XContentHelper.createParser( NamedXContentRegistry.EMPTY, diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java index 2301dd1928b42..93f5a44c7bdd3 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -508,7 +509,8 @@ public static InputStream filterToXContent(HttpRequest request, XContentType xCo String excludeField) throws IOException { try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); XContentBuilder filteredBuilder = new XContentBuilder(xContentType.xContent(), bos, - Collections.emptySet(), Collections.singleton(excludeField), xContentType.mediaType())) { + Collections.emptySet(), Collections.singleton(excludeField), + ParsedMediaType.parseMediaType(xContentType.mediaType()))) { request.toXContent(filteredBuilder, params); filteredBuilder.flush(); return new ByteArrayInputStream(bos.toByteArray()); From 01ce0f4914a433f15a231cf7af0b4a9acdd2aef6 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 17 Dec 2020 16:45:20 +0100 Subject: [PATCH 24/29] fix tests and new test for request/response headers --- .../common/xcontent/ParsedMediaType.java | 2 +- x-pack/plugin/rest-compatibility/build.gradle | 1 + .../compat/ResponseContentTypeHeaderIT.java | 99 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java index 6073f91ef89f8..a7db4608bc867 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -107,7 +107,7 @@ public static ParsedMediaType parseMediaType(String headerValue) { } public static ParsedMediaType parseMediaType(XContentType requestContentType, Map parameters) { - ParsedMediaType parsedMediaType = parseMediaType(requestContentType.mediaType()); + ParsedMediaType parsedMediaType = parseMediaType(requestContentType.mediaTypeWithoutParameters()); parsedMediaType.parameters.putAll(parameters); return parsedMediaType; } diff --git a/x-pack/plugin/rest-compatibility/build.gradle b/x-pack/plugin/rest-compatibility/build.gradle index 7674be749f5fa..2867e0e438dc4 100644 --- a/x-pack/plugin/rest-compatibility/build.gradle +++ b/x-pack/plugin/rest-compatibility/build.gradle @@ -18,6 +18,7 @@ */ apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.rest-test' esplugin { name 'rest-compatibility' diff --git a/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java b/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java new file mode 100644 index 0000000000000..941e51d52b038 --- /dev/null +++ b/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.compat; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; + +import static org.hamcrest.core.IsEqual.equalTo; + +public class ResponseContentTypeHeaderIT extends ESRestTestCase { + + public void testResponseContentType() throws IOException { + Request request = new Request("GET", "/"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=" + Version.CURRENT.major); + request.setOptions(options); + Response response = client().performRequest(request); + + assertThat( + response.getHeader("Content-Type"), + equalTo("application/vnd.elasticsearch+json;compatible-with=" + Version.CURRENT.major) + ); + } + + public void testRequestContentType() throws IOException { + Request request = new Request("PUT", "/sample_index_name"); + Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build(); + String entity = "{\"settings\": " + Strings.toString(settings) + "}"; + StringEntity stringEntity = new StringEntity( + entity, + ContentType.parse("application/vnd.elasticsearch+json;compatible-with=" + Version.CURRENT.major) + ); + request.setEntity(stringEntity); + + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=" + Version.CURRENT.major); + request.setOptions(options); + + Response response = client().performRequest(request); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); + } + + public void testInvalidRequestContentType() throws IOException { + Request request = new Request("PUT", "/sample_index_name"); + Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build(); + String entity = "{\"settings\": " + Strings.toString(settings) + "}"; + StringEntity stringEntity = new StringEntity( + entity, + ContentType.parse("application/vnd.elasticsearch+xxx;compatible-with=" + Version.CURRENT.major) + ); + request.setEntity(stringEntity); + + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=" + Version.CURRENT.major); + request.setOptions(options); + + ResponseException exc = expectThrows(ResponseException.class, () -> client().performRequest(request)); + + assertThat(exc.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.NOT_ACCEPTABLE.getStatus())); + assertTrue( + exc.getMessage().contains("Content-Type header [application/vnd.elasticsearch+xxx; compatible-with=8] is not supported") + ); + } +} From d3ebf1f49fdff8551d00846de0c479f87ec5678f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 17 Dec 2020 18:03:22 +0100 Subject: [PATCH 25/29] replace application/json;charset=utf-8 --- .../common/xcontent/XContentType.java | 2 +- .../script/mustache/CustomMustacheFactory.java | 2 +- .../admin/indices/RestGetAliasesActionTests.java | 14 +++++++------- .../elasticsearch/script/StoredScriptTests.java | 2 +- .../xpack/watcher/input/http/HttpInputTests.java | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index baaae87ef0e2e..de947a5bf4541 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -43,7 +43,7 @@ public String mediaTypeWithoutParameters() { @Override public String mediaType() { - return "application/json; charset=UTF-8"; + return "application/json;charset=utf-8"; } @Override diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java index 9750f3918f0d6..f47ab68a940d9 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java @@ -52,7 +52,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory { - static final String JSON_MIME_TYPE_WITH_CHARSET = "application/json; charset=UTF-8"; + static final String JSON_MIME_TYPE_WITH_CHARSET = "application/json;charset=utf-8"; static final String JSON_MIME_TYPE = "application/json"; static final String PLAIN_TEXT_MIME_TYPE = "text/plain"; static final String X_WWW_FORM_URLENCODED_MIME_TYPE = "application/x-www-form-urlencoded"; diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java index 48c43035488a9..cf84c323ca972 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesActionTests.java @@ -54,7 +54,7 @@ public void testBareRequest() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(false, new String[0], openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{},\"foobar\":{}}}}")); } @@ -64,7 +64,7 @@ public void testSimpleAliasWildcardMatchingNothing() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*" }, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); } @@ -76,7 +76,7 @@ public void testMultipleAliasWildcardsSomeMatching() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "baz*", "foobar*" }, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foobar\":{}}}}")); } @@ -86,7 +86,7 @@ public void testAliasWildcardsIncludeAndExcludeAll() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foob*", "-foo*" }, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); } @@ -98,7 +98,7 @@ public void testAliasWildcardsIncludeAndExcludeSome() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo*", "-foob*" }, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"index\":{\"aliases\":{\"foo\":{}}}}")); } @@ -117,7 +117,7 @@ public void testAliasWildcardsIncludeAndExcludeSomeAndExplicitMissing() throws E final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, aliasPattern, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(NOT_FOUND)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{\"error\":\"alias [missing] missing\",\"status\":404,\"index\":{\"aliases\":{\"foo\":{}}}}")); } @@ -128,7 +128,7 @@ public void testAliasWildcardsExcludeExplicitMissing() throws Exception { final RestResponse restResponse = RestGetAliasesAction.buildRestResponse(true, new String[] { "foo", "foofoo", "-foo*" }, openMapBuilder.build(), xContentBuilder); assertThat(restResponse.status(), equalTo(OK)); - assertThat(restResponse.contentType(), equalTo("application/json; charset=UTF-8")); + assertThat(restResponse.contentType(), equalTo("application/json;charset=utf-8")); assertThat(restResponse.content().utf8ToString(), equalTo("{}")); } } diff --git a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java index 3f8fcb829b0de..8b3602a1a84ef 100644 --- a/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/StoredScriptTests.java @@ -80,7 +80,7 @@ public void testSourceParsing() throws Exception { StoredScriptSource parsed = StoredScriptSource.parse(BytesReference.bytes(builder), XContentType.JSON); StoredScriptSource source = new StoredScriptSource("mustache", code, - Collections.singletonMap("content_type", "application/json; charset=UTF-8")); + Collections.singletonMap("content_type", "application/json;charset=utf-8")); assertThat(parsed, equalTo(source)); } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java index 42788b2ea7ad1..001d2dc1ef7cf 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/input/http/HttpInputTests.java @@ -267,7 +267,7 @@ public void testThatExpectedContentTypeOverridesReturnedContentType() throws Exc ExecutableHttpInput input = new ExecutableHttpInput(httpInput, httpClient, templateEngine); Map headers = new HashMap<>(1); - String contentType = randomFrom("application/json", "application/json; charset=UTF-8", "text/html", "application/yaml", + String contentType = randomFrom("application/json", "application/json;charset=utf-8", "text/html", "application/yaml", "application/smile", "application/cbor"); headers.put("Content-Type", new String[] { contentType }); String body = "{\"foo\":\"bar\"}"; From e1d0463bf7f0fcf8a7d12afe02a2bc9c5c0db9db Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 17 Dec 2020 18:16:26 +0100 Subject: [PATCH 26/29] file header and ;; --- .../xpack/core/ml/action/PostDataAction.java | 2 +- .../compat/ResponseContentTypeHeaderIT.java | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java index bd05a1e367af4..1ce8bf4a2319c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostDataAction.java @@ -112,7 +112,7 @@ public Request(StreamInput in) throws IOException { dataDescription = in.readOptionalWriteable(DataDescription::new); content = in.readBytesReference(); if (in.readBoolean()) { - xContentType = in.readEnum(XContentType.class);; + xContentType = in.readEnum(XContentType.class); } } diff --git a/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java b/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java index 941e51d52b038..9800fc44843db 100644 --- a/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java +++ b/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java @@ -4,25 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.elasticsearch.compat; import org.apache.http.entity.ContentType; From e02970cc7435a8b5b379e5c344fba560846f9caa Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 18 Dec 2020 10:30:10 +0100 Subject: [PATCH 27/29] javadoc, it test in javaRestTest and scripting response about charset --- docs/reference/search/search-template.asciidoc | 2 +- .../elasticsearch/common/xcontent/XContentType.java | 9 ++++++--- x-pack/plugin/rest-compatibility/build.gradle | 10 +++++++++- .../compat/ResponseContentTypeHeaderIT.java | 0 4 files changed, 16 insertions(+), 5 deletions(-) rename x-pack/plugin/rest-compatibility/src/{test => javaRestTest}/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java (100%) diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc index cf0c78bf5496d..fbbdf10e6d0a0 100644 --- a/docs/reference/search/search-template.asciidoc +++ b/docs/reference/search/search-template.asciidoc @@ -158,7 +158,7 @@ The API returns the following result: "lang" : "mustache", "source" : """{"query":{"match":{"title":"{{query_string}}"}}}""", "options": { - "content_type" : "application/json; charset=UTF-8" + "content_type" : "application/json;charset=utf-8" } }, "_id": "", diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index de947a5bf4541..f8e0dc23f0233 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -325,9 +325,12 @@ public String mediaType() { public abstract String mediaTypeWithoutParameters(); /** - * Returns a canonical XContentType for given this XContentType. - * XContentTypes can derive from other XContentType. Like application/vnd.elasticsearch+json derives from application/json - * A canonical is the XContentType that is being derived from. + * Returns a canonical XContentType for this XContentType. + * A canonical XContentType is used to serialize or deserialize the data from/to for HTTP. + * More specialized XContentType types such as vnd* variants still use the general data structure, + * but may have semantic differences. + * Example: XContentType.VND_JSON has a canonical XContentType.JSON + * XContentType.JSON has a canonical XContentType.JSON */ public XContentType canonical(){ return this; diff --git a/x-pack/plugin/rest-compatibility/build.gradle b/x-pack/plugin/rest-compatibility/build.gradle index 2867e0e438dc4..e6c16ee1ffd4f 100644 --- a/x-pack/plugin/rest-compatibility/build.gradle +++ b/x-pack/plugin/rest-compatibility/build.gradle @@ -18,7 +18,7 @@ */ apply plugin: 'elasticsearch.esplugin' -apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.java-rest-test' esplugin { name 'rest-compatibility' @@ -31,3 +31,11 @@ dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') } + + +testClusters.all { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' +} diff --git a/x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java b/x-pack/plugin/rest-compatibility/src/javaRestTest/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java similarity index 100% rename from x-pack/plugin/rest-compatibility/src/test/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java rename to x-pack/plugin/rest-compatibility/src/javaRestTest/java/org/elasticsearch/compat/ResponseContentTypeHeaderIT.java From 1f28dfd665a8756a837b6cbe4e22b4ce6a97be76 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 29 Dec 2020 14:29:37 +0100 Subject: [PATCH 28/29] Make scripting use media type parsing --- .../common/xcontent/MediaType.java | 4 +- .../common/xcontent/MediaTypeRegistry.java | 17 +++-- .../common/xcontent/ParsedMediaType.java | 8 ++- .../common/xcontent/XContentBuilder.java | 4 +- .../common/xcontent/XContentType.java | 34 ++++----- .../mustache/CustomMustacheFactory.java | 47 ++++++++----- .../mustache/CustomMustacheFactoryTests.java | 56 ++++++++++----- .../script/MustacheMediaType.java | 69 +++++++++++++++++++ .../java/org/elasticsearch/script/Script.java | 9 ++- .../rest/RestControllerTests.java | 4 +- .../org/elasticsearch/script/ScriptTests.java | 20 ++++-- .../xpack/sql/plugin/TextFormat.java | 13 ++-- 12 files changed, 201 insertions(+), 84 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/script/MustacheMediaType.java diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java index 2221f1bc97614..0cf1a609e9128 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaType.java @@ -22,8 +22,8 @@ import org.elasticsearch.common.collect.Tuple; import java.util.Collections; +import java.util.List; import java.util.Map; -import java.util.Set; /** * Abstracts a Media Type and a query parameter format. @@ -44,7 +44,7 @@ public interface MediaType { * Returns a set of HeaderValues - allowed media type values on Accept or Content-Type headers * Also defines media type parameters for validation. */ - Set headerValues(); + List headerValues(); /** * A class to represent supported mediaType values i.e. application/json and parameters to be validated. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java index 674770944516d..c0ca282b12762 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/MediaTypeRegistry.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.regex.Pattern; /** @@ -63,12 +62,16 @@ public Map parametersFor(String typeWithSubtype) { public MediaTypeRegistry register(T[] mediaTypes ) { for (T mediaType : mediaTypes) { - Set tuples = mediaType.headerValues(); - for (MediaType.HeaderValue headerValue : tuples) { - queryParamToMediaType.put(mediaType.queryParameter(), mediaType); - typeWithSubtypeToMediaType.put(headerValue.v1(), mediaType); - parametersMap.put(headerValue.v1(), convertPatterns(headerValue.v2())); - } + register(mediaType); + } + return this; + } + + public MediaTypeRegistry register(T mediaType) { + for (MediaType.HeaderValue headerValue : mediaType.headerValues()) { + queryParamToMediaType.put(mediaType.queryParameter(), mediaType); + typeWithSubtypeToMediaType.put(headerValue.v1(), mediaType); + parametersMap.put(headerValue.v1(), convertPatterns(headerValue.v2())); } return this; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java index a7db4608bc867..21cf1b29543c2 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ParsedMediaType.java @@ -106,8 +106,8 @@ public static ParsedMediaType parseMediaType(String headerValue) { return null; } - public static ParsedMediaType parseMediaType(XContentType requestContentType, Map parameters) { - ParsedMediaType parsedMediaType = parseMediaType(requestContentType.mediaTypeWithoutParameters()); + public static ParsedMediaType parseMediaType(MediaType requestContentType, Map parameters) { + ParsedMediaType parsedMediaType = parseMediaType(requestContentType.headerValues().get(0).v1()); parsedMediaType.parameters.putAll(parameters); return parsedMediaType; } @@ -159,6 +159,10 @@ public String toString() { return originalHeaderValue; } + public String responseContentTypeHeader() { + return responseContentTypeHeader(this.parameters); + } + public String responseContentTypeHeader(Map parameters) { return this.mediaTypeWithoutParameters() + formatParameters(parameters); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index b19235d3e8060..59d6b320415b4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -208,9 +208,7 @@ public XContentBuilder(XContent xContent, OutputStream os, Set includes, } public String getResponseContentTypeString() { - Map parameters = responseContentType != null ? - responseContentType.getParameters() : Collections.emptyMap(); - return responseContentType.responseContentTypeHeader(parameters); + return responseContentType.responseContentTypeHeader(); } public XContentType contentType() { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index f8e0dc23f0233..8660e20c6da12 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.xcontent.smile.SmileXContent; import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import java.util.List; import java.util.Map; -import java.util.Set; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -57,8 +57,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue("application/json"), new HeaderValue("application/x-ndjson"), new HeaderValue("application/*")); @@ -84,8 +84,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue("application/smile")); } }, @@ -109,8 +109,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue("application/yaml")); } }, @@ -134,8 +134,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue("application/cbor")); } }, @@ -159,8 +159,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(VENDOR_APPLICATION_PREFIX + "json", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN)), new HeaderValue(VENDOR_APPLICATION_PREFIX + "x-ndjson", @@ -192,8 +192,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(VENDOR_APPLICATION_PREFIX + "smile", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } @@ -223,8 +223,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(VENDOR_APPLICATION_PREFIX + "yaml", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } @@ -254,8 +254,8 @@ public XContent xContent() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(VENDOR_APPLICATION_PREFIX + "cbor", Map.of(COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java index f47ab68a940d9..0e29450908a83 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java @@ -30,10 +30,12 @@ import com.github.mustachejava.codes.DefaultMustache; import com.github.mustachejava.codes.IterableCode; import com.github.mustachejava.codes.WriteCode; -import org.apache.lucene.search.highlight.DefaultEncoder; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.script.MustacheMediaType; import java.io.IOException; import java.io.StringWriter; @@ -51,28 +53,32 @@ import java.util.regex.Pattern; public class CustomMustacheFactory extends DefaultMustacheFactory { + private static final MediaType DEFAULT_MIME_TYPE = XContentType.JSON; - static final String JSON_MIME_TYPE_WITH_CHARSET = "application/json;charset=utf-8"; - static final String JSON_MIME_TYPE = "application/json"; - static final String PLAIN_TEXT_MIME_TYPE = "text/plain"; - static final String X_WWW_FORM_URLENCODED_MIME_TYPE = "application/x-www-form-urlencoded"; - - private static final String DEFAULT_MIME_TYPE = JSON_MIME_TYPE; - - private static final Map> ENCODERS = Map.of( - JSON_MIME_TYPE_WITH_CHARSET, JsonEscapeEncoder::new, - JSON_MIME_TYPE, JsonEscapeEncoder::new, - PLAIN_TEXT_MIME_TYPE, DefaultEncoder::new, - X_WWW_FORM_URLENCODED_MIME_TYPE, UrlEncoder::new); + private static final Map> ENCODERS = Map.of( + XContentType.JSON, JsonEscapeEncoder::new, + MustacheMediaType.PLAIN_TEXT, DefaultEncoder::new, + MustacheMediaType.X_WWW_FORM_URLENCODED, UrlEncoder::new); private final Encoder encoder; - public CustomMustacheFactory(String mimeType) { - super(); + public CustomMustacheFactory(MediaType mimeType) { setObjectHandler(new CustomReflectionObjectHandler()); this.encoder = createEncoder(mimeType); } + public CustomMustacheFactory(String mimeType) { + this(parseMediaType(mimeType)); + } + + private static MediaType parseMediaType(String mimeType) { + MediaType mustacheMediaType = ParsedMediaType.parseMediaType(mimeType).toMediaType(MustacheMediaType.REGISTRY); + if (mustacheMediaType == null) { + throw new IllegalArgumentException("No encoder found for MIME type [" + mimeType + "]"); + } + return mustacheMediaType; + } + public CustomMustacheFactory() { this(DEFAULT_MIME_TYPE); } @@ -86,14 +92,19 @@ public void encode(String value, Writer writer) { } } - static Encoder createEncoder(String mimeType) { - final Supplier supplier = ENCODERS.get(mimeType); + private static Encoder createEncoder(MediaType mustacheMediaType) { + final Supplier supplier = ENCODERS.get(mustacheMediaType); if (supplier == null) { - throw new IllegalArgumentException("No encoder found for MIME type [" + mimeType + "]"); + throw new IllegalArgumentException("No encoder found for MIME type [" + mustacheMediaType + "]"); } return supplier.get(); } + // package scope for testing + Encoder getEncoder() { + return encoder; + } + @Override public MustacheVisitor createMustacheVisitor() { return new CustomMustacheVisitor(this); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java index f5465f256c225..c2cfc96138a73 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java @@ -19,54 +19,74 @@ package org.elasticsearch.script.mustache; +import org.elasticsearch.common.xcontent.ParsedMediaType; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.script.MustacheMediaType; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.TemplateScript; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; import java.util.Map; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; -import static org.elasticsearch.script.mustache.CustomMustacheFactory.JSON_MIME_TYPE; -import static org.elasticsearch.script.mustache.CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE; -import static org.elasticsearch.script.mustache.CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class CustomMustacheFactoryTests extends ESTestCase { + String JSON_MIME_TYPE_WITH_CHARSET = ParsedMediaType.parseMediaType(XContentType.JSON, Map.of("charset", "utf-8")) + .responseContentTypeHeader(); + String JSON_MIME_TYPE = ParsedMediaType.parseMediaType(XContentType.JSON, Collections.emptyMap()) + .responseContentTypeHeader(); + String PLAIN_TEXT_MIME_TYPE = ParsedMediaType.parseMediaType(MustacheMediaType.PLAIN_TEXT, Collections.emptyMap()) + .responseContentTypeHeader(); + String X_WWW_FORM_URLENCODED_MIME_TYPE = ParsedMediaType.parseMediaType(MustacheMediaType.X_WWW_FORM_URLENCODED, Collections.emptyMap()) + .responseContentTypeHeader(); public void testCreateEncoder() { { final IllegalArgumentException e = - expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("non-existent")); - assertThat(e.getMessage(), equalTo("No encoder found for MIME type [non-existent]")); + expectThrows(IllegalArgumentException.class, () -> new CustomMustacheFactory("non-existent")); + assertThat(e.getMessage(), equalTo("invalid media-type [non-existent]")); } { - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("")); - assertThat(e.getMessage(), equalTo("No encoder found for MIME type []")); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new CustomMustacheFactory("")); + assertThat(e.getMessage(), equalTo("invalid media-type []")); } { final IllegalArgumentException e = - expectThrows(IllegalArgumentException.class, () -> CustomMustacheFactory.createEncoder("test")); - assertThat(e.getMessage(), equalTo("No encoder found for MIME type [test]")); + expectThrows(IllegalArgumentException.class, () -> new CustomMustacheFactory("test")); + assertThat(e.getMessage(), equalTo("invalid media-type [test]")); } - assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE_WITH_CHARSET), + String contentType = ParsedMediaType.parseMediaType("application/json ; charset=UTF-8") + .responseContentTypeHeader(); + assertThat(new CustomMustacheFactory(contentType).getEncoder(), + instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); + assertThat(new CustomMustacheFactory(JSON_MIME_TYPE_WITH_CHARSET).getEncoder(), + instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); + assertThat(new CustomMustacheFactory(JSON_MIME_TYPE).getEncoder(), + instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); + + assertThat(new CustomMustacheFactory(PLAIN_TEXT_MIME_TYPE).getEncoder(), + instanceOf(CustomMustacheFactory.DefaultEncoder.class)); + assertThat(new CustomMustacheFactory(X_WWW_FORM_URLENCODED_MIME_TYPE).getEncoder(), + instanceOf(CustomMustacheFactory.UrlEncoder.class)); + + //TODO PG should this be rejected? + contentType = ParsedMediaType.parseMediaType("application/json ; unknown=UTF-8") + .responseContentTypeHeader(); + assertThat(new CustomMustacheFactory(contentType).getEncoder(), instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); - assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE), - instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); - assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE), - instanceOf(CustomMustacheFactory.DefaultEncoder.class)); - assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE), - instanceOf(CustomMustacheFactory.UrlEncoder.class)); } public void testJsonEscapeEncoder() { final ScriptEngine engine = new MustacheScriptEngine(); - final Map params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MIME_TYPE) : emptyMap(); + final Map params = randomBoolean() ? singletonMap(Script.CONTENT_TYPE_OPTION, JSON_MIME_TYPE) : + Collections.emptyMap(); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); diff --git a/server/src/main/java/org/elasticsearch/script/MustacheMediaType.java b/server/src/main/java/org/elasticsearch/script/MustacheMediaType.java new file mode 100644 index 0000000000000..0344f3e233c88 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/MustacheMediaType.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.MediaTypeRegistry; +import org.elasticsearch.common.xcontent.XContentType; + +import java.util.List; + +public enum MustacheMediaType implements MediaType { + + // JSON { +// @Override +// public String queryParameter() { +// return null; +// } +// +// @Override +// public List headerValues() { +// return List.of( +// new HeaderValue("application/json", +// Map.of("charset", "utf-8"))); +// } +// }, + PLAIN_TEXT { + @Override + public String queryParameter() { + return null; + } + + @Override + public List headerValues() { + return List.of(new HeaderValue("text/plain")); + } + }, + X_WWW_FORM_URLENCODED { + @Override + public String queryParameter() { + return null; + } + + @Override + public List headerValues() { + return List.of(new HeaderValue("application/x-www-form-urlencoded")); + } + }; + + public static final MediaTypeRegistry REGISTRY = new MediaTypeRegistry<>() + .register(MustacheMediaType.values()) + .register(XContentType.JSON); // TODO PG make JSON and ND-JSON separate? then make the headerValues return single instance? +} diff --git a/server/src/main/java/org/elasticsearch/script/Script.java b/server/src/main/java/org/elasticsearch/script/Script.java index 1b5566c5ca1c5..b3f3f3e96f944 100644 --- a/server/src/main/java/org/elasticsearch/script/Script.java +++ b/server/src/main/java/org/elasticsearch/script/Script.java @@ -30,9 +30,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.AbstractObjectParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -631,9 +633,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) builder.startObject(); String contentType = options == null ? null : options.get(CONTENT_TYPE_OPTION); - - if (type == ScriptType.INLINE) { - if (contentType != null && builder.contentType().mediaType().equals(contentType)) { + ParsedMediaType parsedMediaType = ParsedMediaType.parseMediaType(contentType); + MediaType type = parsedMediaType != null ? parsedMediaType.toMediaType(MustacheMediaType.REGISTRY) : null; + if (this.type == ScriptType.INLINE) { + if (contentType != null && builder.contentType().equals(type)) { try (InputStream stream = new BytesArray(idOrCode).streamInput()) { builder.rawField(SOURCE_PARSE_FIELD.getPreferredName(), stream); } diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 5683352c84d14..b5612724f44a7 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -732,8 +732,8 @@ public Version compatibleWithVersion() { private String randomCompatibleMimeType(byte version) { XContentType type = randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML); - return ParsedMediaType.parseMediaType(type.mediaType()) - .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version))); + return ParsedMediaType.parseMediaType(type, Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version))) + .responseContentTypeHeader(); } private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements diff --git a/server/src/test/java/org/elasticsearch/script/ScriptTests.java b/server/src/test/java/org/elasticsearch/script/ScriptTests.java index 58560b1d1a85a..d459beaa1998d 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptTests.java @@ -44,7 +44,7 @@ public class ScriptTests extends ESTestCase { public void testScriptParsing() throws IOException { - Script expectedScript = createScript(); + Script expectedScript = createScript(XContentType.JSON.mediaType()); try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser parser = createParser(builder)) { @@ -55,7 +55,17 @@ public void testScriptParsing() throws IOException { } public void testScriptSerialization() throws IOException { - Script expectedScript = createScript(); + Script expectedScript = createScript(XContentType.JSON.mediaType()); + testSerialisation(expectedScript); + + expectedScript = createScript("application/json;charset=utf-8"); + testSerialisation(expectedScript); + + expectedScript = createScript("application/json; charset=UTF-8"); + testSerialisation(expectedScript); + } + + private void testSerialisation(Script expectedScript) throws IOException { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { expectedScript.writeTo(new OutputStreamStreamOutput(out)); try (ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray())) { @@ -65,7 +75,7 @@ public void testScriptSerialization() throws IOException { } } - private Script createScript() throws IOException { + private Script createScript(String contentType) throws IOException { final Map params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value"); ScriptType scriptType = randomFrom(ScriptType.values()); String script; @@ -84,12 +94,12 @@ private Script createScript() throws IOException { scriptType == ScriptType.STORED ? null : randomFrom("_lang1", "_lang2", "_lang3"), script, scriptType == ScriptType.INLINE ? - Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()) : null, params + Collections.singletonMap(Script.CONTENT_TYPE_OPTION, contentType) : null, params ); } public void testParse() throws IOException { - Script expectedScript = createScript(); + Script expectedScript = createScript(XContentType.JSON.mediaType()); try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) { expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS); try (XContentParser xParser = createParser(builder)) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java index 6b771dcfd7ca3..7eb36de7325e9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TextFormat.java @@ -26,7 +26,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.Function; import static org.elasticsearch.xpack.sql.action.BasicFormatter.FormatOption.TEXT; @@ -105,8 +104,8 @@ protected String eol() { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(CONTENT_TYPE_TXT, Map.of("header", "present|absent")), new HeaderValue(VENDOR_CONTENT_TYPE_TXT, @@ -229,8 +228,8 @@ boolean hasHeader(RestRequest request) { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(CONTENT_TYPE_CSV, Map.of("header", "present|absent","delimiter", ".+")),// more detailed parsing is in TextFormat.CSV#delimiter new HeaderValue(VENDOR_CONTENT_TYPE_CSV, @@ -288,8 +287,8 @@ String maybeEscape(String value, Character __) { } @Override - public Set headerValues() { - return Set.of( + public List headerValues() { + return List.of( new HeaderValue(CONTENT_TYPE_TSV, Map.of("header", "present|absent")), new HeaderValue(VENDOR_CONTENT_TYPE_TSV, Map.of("header", "present|absent", COMPATIBLE_WITH_PARAMETER_NAME, VERSION_PATTERN))); From a4fadb4d5a5a5b9db656e94fea456f82e20db555 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 5 Jan 2021 11:56:44 +0100 Subject: [PATCH 29/29] unmute --- .../resources/rest-api-spec/test/mixed_cluster/10_basic.yml | 4 ---- .../rest-api-spec/test/upgraded_cluster/10_basic.yml | 3 --- 2 files changed, 7 deletions(-) diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index b79f73639da4c..bc4067776a045 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -1,9 +1,5 @@ --- "Verify that we can still find things with the template": - - skip: - version: "all" - reason: "Awaits fix: https://github.com/elastic/elasticsearch/issues/66986" - - do: search_template: rest_total_hits_as_int: true diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 16da4c9eb7f0b..33e3e2ede5157 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -1,8 +1,5 @@ --- "Verify that we can still find things with the template": - - skip: - version: "all" - reason: "Awaits fix: https://github.com/elastic/elasticsearch/issues/66986" - do: search_template: