Optionally require a valid content type for all rest requests with content (#22691)

This change adds a strict mode for xcontent parsing on the rest layer. The strict mode will be off by default for 5.x
and in a separate commit will be enabled by default for 6.0. The strict mode, which can be enabled by setting
`http.content_type.required: true` in 5.x, will require that all incoming rest requests have a valid and supported
content type header before the request is dispatched. In the non-strict mode, the Content-Type header will be inspected
and if it is not present or not valid, we will continue with auto detection of content like we have done previously.

The content type header is parsed to the matching XContentType value with the only exception being for plain text
requests. This value is then passed on with the content bytes so that we can reduce the number of places where we need
to auto-detect the content type.

As part of this, many transport requests and builders were updated to provide methods that accepted the XContentType
along with the bytes and the methods that would rely on auto-detection have been deprecated.

In the non-strict mode, deprecation warnings are issued whenever a request with body doesn't provide the Content-Type
header.

See #19388

Author: jaymode

client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugin.noop.NoopPlugin;
 import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction;
@@ -80,7 +81,7 @@ private static final class TransportBulkRequestExecutor implements BulkRequestEx
         public boolean bulkIndex(List<String> bulkData) {
             NoopBulkRequestBuilder builder = NoopBulkAction.INSTANCE.newRequestBuilder(client);
             for (String bulkItem : bulkData) {
-                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8)));
+                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
             }
             BulkResponse bulkResponse;
             try {

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class NoopBulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkResponse, NoopBulkRequestBuilder>
         implements WriteRequestBuilder<NoopBulkRequestBuilder> {
@@ -95,17 +96,17 @@ public NoopBulkRequestBuilder add(UpdateRequestBuilder request) {
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length) throws Exception {
-        request.add(data, from, length, null, null);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception {
+        request.add(data, from, length, null, null, xContentType);
         return this;
     }
 
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType)
-        throws Exception {
-        request.add(data, from, length, defaultIndex, defaultType);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                                      XContentType xContentType) throws Exception {
+        request.add(data, from, length, defaultIndex, defaultType, xContentType);
         return this;
     }
 

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -73,7 +73,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         }
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
-        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true);
+        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true,
+            request.getXContentType());
 
         // short circuit the call to the transport layer
         return channel -> {

core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java

@@ -142,14 +142,28 @@ public PutRepositoryRequest settings(Settings.Builder settings) {
     /**
      * Sets the repository settings.
      *
-     * @param source repository settings in json, yaml or properties format
+     * @param source repository settings in json or yaml format
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection
      */
+    @Deprecated
     public PutRepositoryRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the repository settings.
+     *
+     * @param source repository settings in json or yaml format
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public PutRepositoryRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the repository settings.
      *
@@ -160,7 +174,7 @@ public PutRepositoryRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }

core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -89,16 +90,30 @@ public PutRepositoryRequestBuilder setSettings(Settings.Builder settings) {
     }
 
     /**
-     * Sets the repository settings in Json, Yaml or properties format
+     * Sets the repository settings in Json or Yaml format
      *
      * @param source repository settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection
      */
+    @Deprecated
     public PutRepositoryRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets the repository settings in Json or Yaml format
+     *
+     * @param source repository settings
+     * @param xContentType the contenty type of the source
+     * @return this builder
+     */
+    public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the repository settings
      *

core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java

@@ -83,12 +83,22 @@ public ClusterUpdateSettingsRequest transientSettings(Settings.Builder settings)
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest transientSettings(String source) {
         this.transientSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) {
+        this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -97,7 +107,7 @@ public ClusterUpdateSettingsRequest transientSettings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            transientSettings(builder.string());
+            transientSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -122,12 +132,22 @@ public ClusterUpdateSettingsRequest persistentSettings(Settings.Builder settings
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest persistentSettings(String source) {
         this.persistentSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) {
+        this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */
@@ -136,7 +156,7 @@ public ClusterUpdateSettingsRequest persistentSettings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            persistentSettings(builder.string());
+            persistentSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }

core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -52,12 +53,22 @@ public ClusterUpdateSettingsRequestBuilder setTransientSettings(Settings.Builder
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) {
         request.transientSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) {
+        request.transientSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -84,12 +95,22 @@ public ClusterUpdateSettingsRequestBuilder setPersistentSettings(Settings.Builde
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) {
         request.persistentSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) {
+        request.persistentSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java

@@ -289,18 +289,34 @@ public CreateSnapshotRequest settings(Settings.Builder settings) {
     }
 
     /**
-     * Sets repository-specific snapshot settings in JSON, YAML or properties format
+     * Sets repository-specific snapshot settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public CreateSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>
@@ -313,7 +329,7 @@ public CreateSnapshotRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }

core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -147,12 +148,28 @@ public CreateSnapshotRequestBuilder setSettings(Settings.Builder settings) {
      *
      * @param source repository-specific snapshot settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in YAML or JSON format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>