Add telemetry for flattened fields. (#48972) (#49125)

Currently we just record the number of flattened fields defined in the mappings.

Author: jtibshirani

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsage.java

@@ -6,19 +6,57 @@
 
 package org.elasticsearch.xpack.core.flattened;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class FlattenedFeatureSetUsage extends XPackFeatureSet.Usage {
+    private final int fieldCount;
 
     public FlattenedFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
+        this.fieldCount = input.getVersion().onOrAfter(Version.V_7_6_0) ? input.readInt() : 0;
     }
 
-    public FlattenedFeatureSetUsage(boolean available, boolean enabled) {
+    public FlattenedFeatureSetUsage(boolean available, boolean enabled, int fieldCount) {
         super(XPackField.FLATTENED, available, enabled);
+        this.fieldCount = fieldCount;
+    }
+
+    int fieldCount() {
+        return fieldCount;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_6_0)) {
+            out.writeInt(fieldCount);
+        }
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        builder.field("field_count", fieldCount);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        FlattenedFeatureSetUsage that = (FlattenedFeatureSetUsage) o;
+        return available == that.available && enabled == that.enabled && fieldCount == that.fieldCount;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(available, enabled, fieldCount);
     }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/flattened/FlattenedFeatureSetUsageTests.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.flattened;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.io.IOException;
+
+public class FlattenedFeatureSetUsageTests extends AbstractWireSerializingTestCase<FlattenedFeatureSetUsage> {
+
+    @Override
+    protected FlattenedFeatureSetUsage createTestInstance() {
+        return new FlattenedFeatureSetUsage(randomBoolean(), randomBoolean(), randomIntBetween(0, 1000));
+    }
+
+    @Override
+    protected FlattenedFeatureSetUsage mutateInstance(FlattenedFeatureSetUsage instance) throws IOException {
+
+        boolean available = instance.available();
+        boolean enabled = instance.enabled();
+        int fieldCount = instance.fieldCount();
+
+        switch (between(0, 2)) {
+            case 0:
+                available = !available;
+                break;
+            case 1:
+                enabled = !enabled;
+                break;
+            case 2:
+                fieldCount = randomValueOtherThan(instance.fieldCount(), () -> randomIntBetween(0, 1000));
+                break;
+        }
+
+        return new FlattenedFeatureSetUsage(available, enabled, fieldCount);
+    }
+
+    @Override
+    protected Writeable.Reader<FlattenedFeatureSetUsage> instanceReader() {
+        return FlattenedFeatureSetUsage::new;
+    }
+
+}

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedFeatureSet.java

@@ -6,25 +6,31 @@
 package org.elasticsearch.xpack.flattened;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.XPackFeatureSet;
 import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
+import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper;
 
 import java.util.Map;
 
 public class FlattenedFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
     private final XPackLicenseState licenseState;
+    private final ClusterService clusterService;
 
     @Inject
-    public FlattenedFeatureSet(Settings settings, XPackLicenseState licenseState) {
+    public FlattenedFeatureSet(Settings settings, XPackLicenseState licenseState, ClusterService clusterService) {
         this.enabled = XPackSettings.FLATTENED_ENABLED.get(settings);
         this.licenseState = licenseState;
+        this.clusterService = clusterService;
     }
 
     @Override
@@ -49,6 +55,29 @@ public Map<String, Object> nativeCodeInfo() {
 
     @Override
     public void usage(ActionListener<Usage> listener) {
-        listener.onResponse(new FlattenedFeatureSetUsage(available(), enabled()));
+        int fieldCount = 0;
+        if (available() && enabled() && clusterService.state() != null) {
+            for (IndexMetaData indexMetaData : clusterService.state().metaData()) {
+                MappingMetaData mappingMetaData = indexMetaData.mapping();
+
+                if (mappingMetaData != null) {
+                    Map<String, Object> mappings = mappingMetaData.getSourceAsMap();
+
+                    if (mappings.containsKey("properties")) {
+                        @SuppressWarnings("unchecked")
+                        Map<String, Map<String, Object>> fieldMappings = (Map<String, Map<String, Object>>) mappings.get("properties");
+
+                        for (Map<String, Object> fieldMapping : fieldMappings.values()) {
+                            String fieldType = (String) fieldMapping.get("type");
+                            if (fieldType != null && fieldType.equals(FlatObjectFieldMapper.CONTENT_TYPE)) {
+                                fieldCount++;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        listener.onResponse(new FlattenedFeatureSetUsage(available(), enabled(), fieldCount));
     }
 }

x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/20_flattened_stats.yml

@@ -0,0 +1,41 @@
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: "telemetry for flattened fields was added in 8.0"
+
+---
+"Usage stats for flattened fields":
+  - do:
+      xpack.usage: {}
+
+  - match: { flattened.available: true }
+  - match: { flattened.enabled: true }
+  - match: { flattened.field_count: 0 }
+
+  - do:
+      indices.create:
+        index: test-index1
+        body:
+          mappings:
+            properties:
+              flattened_1:
+                type: flattened
+
+  - do:
+      indices.create:
+        index: test-index2
+        body:
+          mappings:
+            properties:
+              flattened_2:
+                type: flattened
+              flattened_3:
+                type: flattened
+                ignore_above: 10
+
+  - do:
+      xpack.usage: {}
+
+  - match: { flattened.available: true }
+  - match: { flattened.enabled: true }
+  - match: { flattened.field_count: 3 }