Merge master and fix conflicts

.ci/matrix-runtime-javas.yml

@@ -7,7 +7,6 @@
 
 ES_RUNTIME_JAVA:
   - java11
-  - openjdk13
   - openjdk14
   - openjdk15
   - zulu11

build.gradle

@@ -221,8 +221,8 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/issues/53477" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java

@@ -143,7 +143,7 @@ private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
                     repo.patternLayout(layout -> layout.artifact(artifactPattern));
                 });
                 repositories.exclusiveContent(exclusiveContentRepository -> {
-                    exclusiveContentRepository.filter(config -> config.includeGroup(jdk.getVendor()));
+                    exclusiveContentRepository.filter(config -> config.includeGroup(groupName(jdk)));
                     exclusiveContentRepository.forRepositories(ivyRepo);
                 });
             }
@@ -254,7 +254,11 @@ private static String dependencyNotation(Jdk jdk) {
             : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
 
-        return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+        return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
+    }
+
+    private static String groupName(Jdk jdk) {
+        return jdk.getVendor() + "_" + jdk.getMajor();
     }
 
     private static String configName(String... parts) {

buildSrc/version.properties

@@ -11,7 +11,7 @@ jts               = 1.15.0
 # you should also inspect that version to see if it can be advanced along with
 # the com.maxmind.geoip2:geoip2 dependency
 jackson           = 2.8.11
-jacksondatabind   = 2.8.11.4
+jacksondatabind   = 2.8.11.6
 snakeyaml         = 1.17
 icu4j             = 62.1
 supercsv          = 2.4.0

client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/ProxyModeInfo.java

@@ -23,15 +23,18 @@
 
 public class ProxyModeInfo implements RemoteConnectionInfo.ModeInfo {
     static final String NAME = "proxy";
-    static final String ADDRESS = "address";
-    static final String NUM_SOCKETS_CONNECTED = "num_sockets_connected";
-    static final String MAX_SOCKET_CONNECTIONS = "max_socket_connections";
+    static final String PROXY_ADDRESS = "proxy_address";
+    static final String SERVER_NAME = "server_name";
+    static final String NUM_PROXY_SOCKETS_CONNECTED = "num_proxy_sockets_connected";
+    static final String MAX_PROXY_SOCKET_CONNECTIONS = "max_proxy_socket_connections";
     private final String address;
+    private final String serverName;
     private final int maxSocketConnections;
     private final int numSocketsConnected;
 
-    ProxyModeInfo(String address, int maxSocketConnections, int numSocketsConnected) {
+    ProxyModeInfo(String address, String serverName, int maxSocketConnections, int numSocketsConnected) {
         this.address = address;
+        this.serverName = serverName;
         this.maxSocketConnections = maxSocketConnections;
         this.numSocketsConnected = numSocketsConnected;
     }
@@ -50,6 +53,10 @@ public String getAddress() {
         return address;
     }
 
+    public String getServerName() {
+        return serverName;
+    }
+
     public int getMaxSocketConnections() {
         return maxSocketConnections;
     }
@@ -65,11 +72,12 @@ public boolean equals(Object o) {
         ProxyModeInfo otherProxy = (ProxyModeInfo) o;
         return maxSocketConnections == otherProxy.maxSocketConnections &&
                 numSocketsConnected == otherProxy.numSocketsConnected &&
-                Objects.equals(address, otherProxy.address);
+                Objects.equals(address, otherProxy.address) &&
+                Objects.equals(serverName, otherProxy.serverName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(address, maxSocketConnections, numSocketsConnected);
+        return Objects.hash(address, serverName, maxSocketConnections, numSocketsConnected);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/cluster/RemoteConnectionInfo.java

@@ -48,9 +48,9 @@ public final class RemoteConnectionInfo {
                 String mode = (String) args[1];
                 ModeInfo modeInfo;
                 if (mode.equals(ProxyModeInfo.NAME)) {
-                    modeInfo = new ProxyModeInfo((String) args[4], (int) args[5], (int) args[6]);
+                    modeInfo = new ProxyModeInfo((String) args[4], (String) args[5], (int) args[6], (int) args[7]);
                 } else if (mode.equals(SniffModeInfo.NAME)) {
-                    modeInfo = new SniffModeInfo((List<String>) args[7], (int) args[8], (int) args[9]);
+                    modeInfo = new SniffModeInfo((List<String>) args[8], (int) args[9], (int) args[10]);
                 } else {
                     throw new IllegalArgumentException("mode cannot be " + mode);
                 }
@@ -66,9 +66,10 @@ public final class RemoteConnectionInfo {
         PARSER.declareString(constructorArg(), new ParseField(INITIAL_CONNECT_TIMEOUT));
         PARSER.declareBoolean(constructorArg(), new ParseField(SKIP_UNAVAILABLE));
 
-        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.ADDRESS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_SOCKET_CONNECTIONS));
-        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_SOCKETS_CONNECTED));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.PROXY_ADDRESS));
+        PARSER.declareString(optionalConstructorArg(), new ParseField(ProxyModeInfo.SERVER_NAME));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.MAX_PROXY_SOCKET_CONNECTIONS));
+        PARSER.declareInt(optionalConstructorArg(), new ParseField(ProxyModeInfo.NUM_PROXY_SOCKETS_CONNECTED));
 
         PARSER.declareStringArray(optionalConstructorArg(), new ParseField(SniffModeInfo.SEEDS));
         PARSER.declareInt(optionalConstructorArg(), new ParseField(SniffModeInfo.MAX_CONNECTIONS_PER_CLUSTER));

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/Classification.java

@@ -22,10 +22,12 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 
 public class Classification implements DataFrameAnalysis {
@@ -49,6 +51,7 @@ public static Builder builder(String dependentVariable) {
     static final ParseField NUM_TOP_FEATURE_IMPORTANCE_VALUES = new ParseField("num_top_feature_importance_values");
     static final ParseField PREDICTION_FIELD_NAME = new ParseField("prediction_field_name");
     static final ParseField TRAINING_PERCENT = new ParseField("training_percent");
+    static final ParseField CLASS_ASSIGNMENT_OBJECTIVE = new ParseField("class_assignment_objective");
     static final ParseField NUM_TOP_CLASSES = new ParseField("num_top_classes");
     static final ParseField RANDOMIZE_SEED = new ParseField("randomize_seed");
 
@@ -67,7 +70,8 @@ public static Builder builder(String dependentVariable) {
                 (String) a[7],
                 (Double) a[8],
                 (Integer) a[9],
-                (Long) a[10]));
+                (Long) a[10],
+                (ClassAssignmentObjective) a[11]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), DEPENDENT_VARIABLE);
@@ -81,6 +85,12 @@ public static Builder builder(String dependentVariable) {
         PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), TRAINING_PERCENT);
         PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), NUM_TOP_CLASSES);
         PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), RANDOMIZE_SEED);
+        PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
+            if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+                return ClassAssignmentObjective.fromString(p.text());
+            }
+            throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+        }, CLASS_ASSIGNMENT_OBJECTIVE, ObjectParser.ValueType.STRING);
     }
 
     private final String dependentVariable;
@@ -92,13 +102,15 @@ public static Builder builder(String dependentVariable) {
     private final Integer numTopFeatureImportanceValues;
     private final String predictionFieldName;
     private final Double trainingPercent;
+    private final ClassAssignmentObjective classAssignmentObjective;
     private final Integer numTopClasses;
     private final Long randomizeSeed;
 
     private Classification(String dependentVariable, @Nullable Double lambda, @Nullable Double gamma, @Nullable Double eta,
                            @Nullable Integer maxTrees, @Nullable Double featureBagFraction,
                            @Nullable Integer numTopFeatureImportanceValues, @Nullable String predictionFieldName,
-                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed) {
+                           @Nullable Double trainingPercent, @Nullable Integer numTopClasses, @Nullable Long randomizeSeed,
+                           @Nullable ClassAssignmentObjective classAssignmentObjective) {
         this.dependentVariable = Objects.requireNonNull(dependentVariable);
         this.lambda = lambda;
         this.gamma = gamma;
@@ -108,6 +120,7 @@ private Classification(String dependentVariable, @Nullable Double lambda, @Nulla
         this.numTopFeatureImportanceValues = numTopFeatureImportanceValues;
         this.predictionFieldName = predictionFieldName;
         this.trainingPercent = trainingPercent;
+        this.classAssignmentObjective = classAssignmentObjective;
         this.numTopClasses = numTopClasses;
         this.randomizeSeed = randomizeSeed;
     }
@@ -157,6 +170,10 @@ public Long getRandomizeSeed() {
         return randomizeSeed;
     }
 
+    public ClassAssignmentObjective getClassAssignmentObjective() {
+        return classAssignmentObjective;
+    }
+
     public Integer getNumTopClasses() {
         return numTopClasses;
     }
@@ -192,6 +209,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (randomizeSeed != null) {
             builder.field(RANDOMIZE_SEED.getPreferredName(), randomizeSeed);
         }
+        if (classAssignmentObjective != null) {
+            builder.field(CLASS_ASSIGNMENT_OBJECTIVE.getPreferredName(), classAssignmentObjective);
+        }
         if (numTopClasses != null) {
             builder.field(NUM_TOP_CLASSES.getPreferredName(), numTopClasses);
         }
@@ -202,7 +222,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     @Override
     public int hashCode() {
         return Objects.hash(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction, numTopFeatureImportanceValues,
-            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses);
+            predictionFieldName, trainingPercent, randomizeSeed, numTopClasses, classAssignmentObjective);
     }
 
     @Override
@@ -220,14 +240,28 @@ public boolean equals(Object o) {
             && Objects.equals(predictionFieldName, that.predictionFieldName)
             && Objects.equals(trainingPercent, that.trainingPercent)
             && Objects.equals(randomizeSeed, that.randomizeSeed)
-            && Objects.equals(numTopClasses, that.numTopClasses);
+            && Objects.equals(numTopClasses, that.numTopClasses)
+            && Objects.equals(classAssignmentObjective, that.classAssignmentObjective);
     }
 
     @Override
     public String toString() {
         return Strings.toString(this);
     }
 
+    public enum ClassAssignmentObjective {
+        MAXIMIZE_ACCURACY, MAXIMIZE_MINIMUM_RECALL;
+
+        public static ClassAssignmentObjective fromString(String value) {
+            return ClassAssignmentObjective.valueOf(value.toUpperCase(Locale.ROOT));
+        }
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
     public static class Builder {
         private String dependentVariable;
         private Double lambda;
@@ -240,6 +274,7 @@ public static class Builder {
         private Double trainingPercent;
         private Integer numTopClasses;
         private Long randomizeSeed;
+        private ClassAssignmentObjective classAssignmentObjective;
 
         private Builder(String dependentVariable) {
             this.dependentVariable = Objects.requireNonNull(dependentVariable);
@@ -295,9 +330,15 @@ public Builder setNumTopClasses(Integer numTopClasses) {
             return this;
         }
 
+        public Builder setClassAssignmentObjective(ClassAssignmentObjective classAssignmentObjective) {
+            this.classAssignmentObjective = classAssignmentObjective;
+            return this;
+        }
+
         public Classification build() {
             return new Classification(dependentVariable, lambda, gamma, eta, maxTrees, featureBagFraction,
-                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed);
+                numTopFeatureImportanceValues, predictionFieldName, trainingPercent, numTopClasses, randomizeSeed,
+                classAssignmentObjective);
         }
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelDefinition.java

@@ -43,9 +43,8 @@ public class TrainedModelDefinition implements ToXContentObject {
             true,
             TrainedModelDefinition.Builder::new);
     static {
-        PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setTrainedModel,
+        PARSER.declareNamedObject(TrainedModelDefinition.Builder::setTrainedModel,
             (p, c, n) -> p.namedObject(TrainedModel.class, n, null),
-            (modelDocBuilder) -> { /* Noop does not matter client side*/ },
             TRAINED_MODEL);
         PARSER.declareNamedObjects(TrainedModelDefinition.Builder::setPreProcessors,
             (p, c, n) -> p.namedObject(PreProcessor.class, n, null),
@@ -124,11 +123,6 @@ public Builder setTrainedModel(TrainedModel trainedModel) {
             return this;
         }
 
-        private Builder setTrainedModel(List<TrainedModel> trainedModel) {
-            assert trainedModel.size() == 1;
-            return setTrainedModel(trainedModel.get(0));
-        }
-
         public TrainedModelDefinition build() {
             return new TrainedModelDefinition(this.trainedModel, this.preProcessors);
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/Ensemble.java

@@ -56,9 +56,8 @@ public class Ensemble implements TrainedModel {
                     p.namedObject(TrainedModel.class, n, null),
             (ensembleBuilder) -> { /* Noop does not matter client side */ },
             TRAINED_MODELS);
-        PARSER.declareNamedObjects(Ensemble.Builder::setOutputAggregatorFromParser,
+        PARSER.declareNamedObject(Ensemble.Builder::setOutputAggregator,
             (p, c, n) -> p.namedObject(OutputAggregator.class, n, null),
-            (ensembleBuilder) -> { /* Noop does not matter client side */ },
             AGGREGATE_OUTPUT);
         PARSER.declareString(Ensemble.Builder::setTargetType, TARGET_TYPE);
         PARSER.declareStringArray(Ensemble.Builder::setClassificationLabels, CLASSIFICATION_LABELS);
@@ -194,9 +193,6 @@ public Builder setClassificationWeights(List<Double> classificationWeights) {
             return this;
         }
 
-        private void setOutputAggregatorFromParser(List<OutputAggregator> outputAggregators) {
-            this.setOutputAggregator(outputAggregators.get(0));
-        }
 
         private void setTargetType(String targetType) {
             this.targetType = TargetType.fromString(targetType);

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -1336,6 +1336,8 @@ public void testPutDataFrameAnalyticsConfig_GivenClassification() throws Excepti
                 .setPredictionFieldName("my_dependent_variable_prediction")
                 .setTrainingPercent(80.0)
                 .setRandomizeSeed(42L)
+                .setClassAssignmentObjective(
+                    org.elasticsearch.client.ml.dataframe.Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY)
                 .setNumTopClasses(1)
                 .setLambda(1.0)
                 .setGamma(1.0)
@@ -2265,7 +2267,7 @@ public void testGetTrainedModelsStats() throws Exception {
             "          \"target_field\": \"regression_value\",\n" +
             "          \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
             "          \"inference_config\": {\"regression\": {}},\n" +
-            "          \"field_mappings\": {\n" +
+            "          \"field_map\": {\n" +
             "            \"col1\": \"col1\",\n" +
             "            \"col2\": \"col2\",\n" +
             "            \"col3\": \"col3\",\n" +

client/rest-high-level/src/test/java/org/elasticsearch/client/cluster/RemoteInfoResponseTests.java

@@ -83,6 +83,7 @@ protected void assertInstances(org.elasticsearch.action.admin.cluster.remote.Rem
                 ProxyConnectionStrategy.ProxyModeInfo serverModeInfo =
                         (ProxyConnectionStrategy.ProxyModeInfo) serverRemoteInfo.getModeInfo();
                 assertThat(clientModeInfo.getAddress(), equalTo(serverModeInfo.getAddress()));
+                assertThat(clientModeInfo.getServerName(), equalTo(serverModeInfo.getServerName()));
                 assertThat(clientModeInfo.getMaxSocketConnections(), equalTo(serverModeInfo.getMaxSocketConnections()));
                 assertThat(clientModeInfo.getNumSocketsConnected(), equalTo(serverModeInfo.getNumSocketsConnected()));
             } else {
@@ -95,9 +96,10 @@ private static RemoteConnectionInfo createRandomRemoteConnectionInfo() {
         RemoteConnectionInfo.ModeInfo modeInfo;
         if (randomBoolean()) {
             String address = randomAlphaOfLength(8);
+            String serverName = randomAlphaOfLength(8);
             int maxSocketConnections = randomInt(5);
             int numSocketsConnected = randomInt(5);
-            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(address, maxSocketConnections, numSocketsConnected);
+            modeInfo = new ProxyConnectionStrategy.ProxyModeInfo(address, serverName, maxSocketConnections, numSocketsConnected);
         } else {
             List<String> seedNodes = randomList(randomInt(8), () -> randomAlphaOfLength(8));
             int maxConnectionsPerCluster = randomInt(5);

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -139,6 +139,7 @@
 import org.elasticsearch.client.ml.datafeed.DatafeedStats;
 import org.elasticsearch.client.ml.datafeed.DatafeedUpdate;
 import org.elasticsearch.client.ml.datafeed.DelayedDataCheckConfig;
+import org.elasticsearch.client.ml.dataframe.Classification;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalysis;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsDest;
@@ -2969,7 +2970,7 @@ public void testPutDataFrameAnalytics() throws Exception {
             // end::put-data-frame-analytics-outlier-detection-customized
 
             // tag::put-data-frame-analytics-classification
-            DataFrameAnalysis classification = org.elasticsearch.client.ml.dataframe.Classification.builder("my_dependent_variable") // <1>
+            DataFrameAnalysis classification = Classification.builder("my_dependent_variable") // <1>
                 .setLambda(1.0) // <2>
                 .setGamma(5.5) // <3>
                 .setEta(5.5) // <4>
@@ -2979,7 +2980,8 @@ public void testPutDataFrameAnalytics() throws Exception {
                 .setPredictionFieldName("my_prediction_field_name") // <8>
                 .setTrainingPercent(50.0) // <9>
                 .setRandomizeSeed(1234L) // <10>
-                .setNumTopClasses(1) // <11>
+                .setClassAssignmentObjective(Classification.ClassAssignmentObjective.MAXIMIZE_ACCURACY) // <11>
+                .setNumTopClasses(1) // <12>
                 .build();
             // end::put-data-frame-analytics-classification