[ML] Label anomalies with multi_bucket_impact (#34233)

* [ML] Label anomalies with  multi_bucket_impact

Add the multi_bucket_impact field to record results.

Author: edsavage

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/results/AnomalyRecord.java

@@ -48,6 +48,7 @@ public class AnomalyRecord implements ToXContentObject {
      * Result fields (all detector types)
      */
     public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField MULTI_BUCKET_IMPACT = new ParseField("multi_bucket_impact");
     public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
     public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name");
     public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value");
@@ -94,6 +95,7 @@ public class AnomalyRecord implements ToXContentObject {
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
         PARSER.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE);
         PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY);
+        PARSER.declareDouble(AnomalyRecord::setMultiBucketImpact, MULTI_BUCKET_IMPACT);
         PARSER.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE);
         PARSER.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE);
         PARSER.declareInt(AnomalyRecord::setDetectorIndex, DETECTOR_INDEX);
@@ -117,6 +119,7 @@ public class AnomalyRecord implements ToXContentObject {
     private final String jobId;
     private int detectorIndex;
     private double probability;
+    private Double multiBucketImpact;
     private String byFieldName;
     private String byFieldValue;
     private String correlatedByFieldValue;
@@ -155,6 +158,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Job.ID.getPreferredName(), jobId);
         builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
         builder.field(PROBABILITY.getPreferredName(), probability);
+        if (multiBucketImpact != null) {
+            builder.field(MULTI_BUCKET_IMPACT.getPreferredName(), multiBucketImpact);
+        }
         builder.field(RECORD_SCORE.getPreferredName(), recordScore);
         builder.field(INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore);
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
@@ -254,6 +260,14 @@ void setProbability(double value) {
         probability = value;
     }
 
+    public double getMultiBucketImpact() {
+        return multiBucketImpact;
+    }
+
+    void setMultiBucketImpact(double value) {
+        multiBucketImpact = value;
+    }
+
     public String getByFieldName() {
         return byFieldName;
     }
@@ -376,7 +390,7 @@ void setInfluencers(List<Influence> influencers) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(jobId, detectorIndex, bucketSpan, probability, recordScore,
+        return Objects.hash(jobId, detectorIndex, bucketSpan, probability, multiBucketImpact, recordScore,
                 initialRecordScore, typical, actual,function, functionDescription, fieldName,
                 byFieldName, byFieldValue, correlatedByFieldValue, partitionFieldName,
                 partitionFieldValue, overFieldName, overFieldValue, timestamp, isInterim,
@@ -399,6 +413,7 @@ public boolean equals(Object other) {
             && this.detectorIndex == that.detectorIndex
             && this.bucketSpan == that.bucketSpan
             && this.probability == that.probability
+            && Objects.equals(this.multiBucketImpact, that.multiBucketImpact)
             && this.recordScore == that.recordScore
             && this.initialRecordScore == that.initialRecordScore
             && Objects.deepEquals(this.typical, that.typical)

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/AnomalyRecordTests.java

@@ -38,6 +38,9 @@ public static AnomalyRecord createTestInstance(String jobId) {
         anomalyRecord.setActual(Collections.singletonList(randomDouble()));
         anomalyRecord.setTypical(Collections.singletonList(randomDouble()));
         anomalyRecord.setProbability(randomDouble());
+        if (randomBoolean()) {
+            anomalyRecord.setMultiBucketImpact(randomDouble());
+        }
         anomalyRecord.setRecordScore(randomDouble());
         anomalyRecord.setInitialRecordScore(randomDouble());
         anomalyRecord.setInterim(randomBoolean());

docs/reference/ml/apis/resultsresource.asciidoc

@@ -364,6 +364,11 @@ A record object has the following properties:
 //In scientific notation, a value of 3.24E-300 is highly unlikely and therefore
 //highly anomalous.
 
+`multi_bucket_impact`::
+  (number) an indication of how strongly an anomaly is multi bucket or single bucket.
+  The value is on a scale of -5 to +5 where -5 means the anomaly is purely single 
+  bucket and +5 means the anomaly is purely multi bucket.
+
 `record_score`::
   (number) A normalized score between 0-100, which is based on the probability
   of the anomalousness of this record. Unlike `initial_record_score`, this

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java

@@ -372,6 +372,9 @@ private static void addAnomalyRecordFieldsToMapping(XContentBuilder builder) thr
         .startObject(AnomalyRecord.PROBABILITY.getPreferredName())
             .field(TYPE, DOUBLE)
         .endObject()
+        .startObject(AnomalyRecord.MULTI_BUCKET_IMPACT.getPreferredName())
+            .field(TYPE, DOUBLE)
+        .endObject()
         .startObject(AnomalyRecord.FUNCTION.getPreferredName())
             .field(TYPE, KEYWORD)
         .endObject()

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.xpack.core.ml.job.config.Job;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.Version;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,6 +45,7 @@ public class AnomalyRecord implements ToXContentObject, Writeable {
      * Result fields (all detector types)
      */
     public static final ParseField PROBABILITY = new ParseField("probability");
+    public static final ParseField MULTI_BUCKET_IMPACT = new ParseField("multi_bucket_impact");
     public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name");
     public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value");
     public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlated_by_field_value");
@@ -100,6 +102,7 @@ private static ConstructingObjectParser<AnomalyRecord, Void> createParser(boolea
         parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN);
         parser.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE);
         parser.declareDouble(AnomalyRecord::setProbability, PROBABILITY);
+        parser.declareDouble(AnomalyRecord::setMultiBucketImpact, MULTI_BUCKET_IMPACT);
         parser.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE);
         parser.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE);
         parser.declareInt(AnomalyRecord::setDetectorIndex, Detector.DETECTOR_INDEX);
@@ -127,6 +130,7 @@ private static ConstructingObjectParser<AnomalyRecord, Void> createParser(boolea
     private final String jobId;
     private int detectorIndex;
     private double probability;
+    private Double multiBucketImpact;
     private String byFieldName;
     private String byFieldValue;
     private String correlatedByFieldValue;
@@ -164,6 +168,9 @@ public AnomalyRecord(StreamInput in) throws IOException {
         jobId = in.readString();
         detectorIndex = in.readInt();
         probability = in.readDouble();
+        if (in.getVersion().onOrAfter(Version.V_6_5_0)) {
+            multiBucketImpact = in.readOptionalDouble();
+        }
         byFieldName = in.readOptionalString();
         byFieldValue = in.readOptionalString();
         correlatedByFieldValue = in.readOptionalString();
@@ -198,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(jobId);
         out.writeInt(detectorIndex);
         out.writeDouble(probability);
+        if (out.getVersion().onOrAfter(Version.V_6_5_0)) {
+            out.writeOptionalDouble(multiBucketImpact);
+        }
         out.writeOptionalString(byFieldName);
         out.writeOptionalString(byFieldValue);
         out.writeOptionalString(correlatedByFieldValue);
@@ -247,6 +257,9 @@ XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws I
         builder.field(Job.ID.getPreferredName(), jobId);
         builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
         builder.field(PROBABILITY.getPreferredName(), probability);
+        if (multiBucketImpact != null) {
+            builder.field(MULTI_BUCKET_IMPACT.getPreferredName(), multiBucketImpact);
+        }
         builder.field(RECORD_SCORE.getPreferredName(), recordScore);
         builder.field(INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore);
         builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan);
@@ -389,6 +402,14 @@ public void setProbability(double value) {
         probability = value;
     }
 
+    public double getMultiBucketImpact() {
+        return multiBucketImpact;
+    }
+
+    public void setMultiBucketImpact(double value) {
+        multiBucketImpact = value;
+    }
+
     public String getByFieldName() {
         return byFieldName;
     }
@@ -519,7 +540,7 @@ public void setInfluencers(List<Influence> influencers) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(jobId, detectorIndex, bucketSpan, probability, recordScore,
+        return Objects.hash(jobId, detectorIndex, bucketSpan, probability, multiBucketImpact, recordScore,
                 initialRecordScore, typical, actual,function, functionDescription, fieldName,
                 byFieldName, byFieldValue, correlatedByFieldValue, partitionFieldName,
                 partitionFieldValue, overFieldName, overFieldValue, timestamp, isInterim,
@@ -543,6 +564,7 @@ public boolean equals(Object other) {
                 && this.detectorIndex == that.detectorIndex
                 && this.bucketSpan == that.bucketSpan
                 && this.probability == that.probability
+                && Objects.equals(this.multiBucketImpact, that.multiBucketImpact)
                 && this.recordScore == that.recordScore
                 && this.initialRecordScore == that.initialRecordScore
                 && Objects.deepEquals(this.typical, that.typical)

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java

@@ -57,6 +57,7 @@ public final class ReservedFieldNames {
             AnomalyCause.FIELD_NAME.getPreferredName(),
 
             AnomalyRecord.PROBABILITY.getPreferredName(),
+            AnomalyRecord.MULTI_BUCKET_IMPACT.getPreferredName(),
             AnomalyRecord.BY_FIELD_NAME.getPreferredName(),
             AnomalyRecord.BY_FIELD_VALUE.getPreferredName(),
             AnomalyRecord.CORRELATED_BY_FIELD_VALUE.getPreferredName(),

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java

@@ -37,6 +37,9 @@ public AnomalyRecord createTestInstance(String jobId) {
         anomalyRecord.setActual(Collections.singletonList(randomDouble()));
         anomalyRecord.setTypical(Collections.singletonList(randomDouble()));
         anomalyRecord.setProbability(randomDouble());
+        if (randomBoolean()) {
+            anomalyRecord.setMultiBucketImpact(randomDouble());
+        }
         anomalyRecord.setRecordScore(randomDouble());
         anomalyRecord.setInitialRecordScore(randomDouble());
         anomalyRecord.setInterim(randomBoolean());