Report overall mapping size in cluster stats (#87556)

Adds measures of the total size of all mappings and the total number of
fields in the cluster (both before and after deduplication).

Relates #86639
Relates #77466

Author: DaveCTurner

docs/changelog/87556.yaml

@@ -0,0 +1,5 @@
+pr: 87556
+summary: Report overall mapping size in cluster stats
+area: Cluster Coordination
+type: enhancement
+issues: []

docs/reference/cluster/stats.asciidoc

@@ -485,6 +485,22 @@ Contains statistics about <<mapping,field mappings>> in selected nodes.
 .Properties of `mappings`
 [%collapsible%open]
 =====
+`total_field_count`::
+(integer)
+Total number of fields in all non-system indices.
+
+`total_deduplicated_field_count`::
+(integer)
+Total number of fields in all non-system indices, accounting for mapping deduplication.
+
+`total_deduplicated_mapping_size`::
+(<<byte-units, byte units>>)
+Total size of all mappings after deduplication and compression.
+
+`total_deduplicated_mapping_size_in_bytes`::
+(integer)
+Total size of all mappings, in bytes, after deduplication and compression.
+
 `field_types`::
 (array of objects)
 Contains statistics about <<mapping-types,field data types>> used in selected
@@ -1363,6 +1379,10 @@ The API returns the following response:
          "file_sizes": {}
       },
       "mappings": {
+        "total_field_count": 0,
+        "total_deduplicated_field_count": 0,
+        "total_deduplicated_mapping_size": "0b",
+        "total_deduplicated_mapping_size_in_bytes": 0,
         "field_types": [],
         "runtime_field_types": []
       },

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml

@@ -166,3 +166,21 @@
   - match: { indices.mappings.runtime_field_types.1.doc_max: 0 }
   - match: { indices.mappings.runtime_field_types.1.doc_total: 0 }
 
+---
+"mappings sizes reported in get cluster stats":
+  - skip:
+      version: " - 8.3.99"
+      reason:  "mapping sizes reported from 8.4 onwards"
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          mappings:
+            "properties":
+              "field":
+                "type": "keyword"
+
+  - do: {cluster.stats: {}}
+  - gt: { indices.mappings.total_field_count: 0 }
+  - gt: { indices.mappings.total_deduplicated_field_count: 0 }
+  - gt: { indices.mappings.total_deduplicated_mapping_size_in_bytes: 0 }

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java

@@ -8,13 +8,16 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -29,7 +32,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.OptionalLong;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -57,6 +62,8 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
             }
             AnalysisStats.countMapping(mappingCounts, indexMetadata);
         }
+        final AtomicLong totalFieldCount = new AtomicLong();
+        final AtomicLong totalDeduplicatedFieldCount = new AtomicLong();
         for (Map.Entry<MappingMetadata, Integer> mappingAndCount : mappingCounts.entrySet()) {
             ensureNotCancelled.run();
             Set<String> indexFieldTypes = new HashSet<>();
@@ -73,6 +80,8 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
                     type = "object";
                 }
                 if (type != null) {
+                    totalDeduplicatedFieldCount.incrementAndGet();
+                    totalFieldCount.addAndGet(count);
                     FieldStats stats;
                     if (type.equals("dense_vector")) {
                         stats = fieldTypes.computeIfAbsent(type, DenseVectorFieldStats::new);
@@ -134,7 +143,17 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) {
                 }
             });
         }
-        return new MappingStats(fieldTypes.values(), runtimeFieldTypes.values());
+        long totalMappingSizeBytes = 0L;
+        for (MappingMetadata mappingMetadata : metadata.getMappingsByHash().values()) {
+            totalMappingSizeBytes += mappingMetadata.source().compressed().length;
+        }
+        return new MappingStats(
+            totalFieldCount.get(),
+            totalDeduplicatedFieldCount.get(),
+            totalMappingSizeBytes,
+            fieldTypes.values(),
+            runtimeFieldTypes.values()
+        );
     }
 
     private static void updateScriptParams(Object scriptSourceObject, FieldScriptStats scriptStats, int multiplier) {
@@ -157,10 +176,28 @@ private static int countOccurrences(String script, Pattern pattern) {
         return occurrences;
     }
 
+    @Nullable // for BwC
+    private final Long totalFieldCount;
+
+    @Nullable // for BwC
+    private final Long totalDeduplicatedFieldCount;
+
+    @Nullable // for BwC
+    private final Long totalMappingSizeBytes;
+
     private final List<FieldStats> fieldTypeStats;
     private final List<RuntimeFieldStats> runtimeFieldStats;
 
-    MappingStats(Collection<FieldStats> fieldTypeStats, Collection<RuntimeFieldStats> runtimeFieldStats) {
+    MappingStats(
+        long totalFieldCount,
+        long totalDeduplicatedFieldCount,
+        long totalMappingSizeBytes,
+        Collection<FieldStats> fieldTypeStats,
+        Collection<RuntimeFieldStats> runtimeFieldStats
+    ) {
+        this.totalFieldCount = totalFieldCount;
+        this.totalDeduplicatedFieldCount = totalDeduplicatedFieldCount;
+        this.totalMappingSizeBytes = totalMappingSizeBytes;
         List<FieldStats> stats = new ArrayList<>(fieldTypeStats);
         stats.sort(Comparator.comparing(IndexFeatureStats::getName));
         this.fieldTypeStats = Collections.unmodifiableList(stats);
@@ -170,16 +207,57 @@ private static int countOccurrences(String script, Pattern pattern) {
     }
 
     MappingStats(StreamInput in) throws IOException {
+        if (in.getVersion().onOrAfter(Version.V_8_4_0)) {
+            totalFieldCount = in.readOptionalVLong();
+            totalDeduplicatedFieldCount = in.readOptionalVLong();
+            totalMappingSizeBytes = in.readOptionalVLong();
+        } else {
+            totalFieldCount = null;
+            totalDeduplicatedFieldCount = null;
+            totalMappingSizeBytes = null;
+        }
         fieldTypeStats = Collections.unmodifiableList(in.readList(FieldStats::new));
         runtimeFieldStats = Collections.unmodifiableList(in.readList(RuntimeFieldStats::new));
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        if (out.getVersion().onOrAfter(Version.V_8_4_0)) {
+            out.writeOptionalVLong(totalFieldCount);
+            out.writeOptionalVLong(totalDeduplicatedFieldCount);
+            out.writeOptionalVLong(totalMappingSizeBytes);
+        } // else just omit these stats, they're not computed on older nodes anyway
         out.writeCollection(fieldTypeStats);
         out.writeCollection(runtimeFieldStats);
     }
 
+    private static OptionalLong ofNullable(Long l) {
+        return l == null ? OptionalLong.empty() : OptionalLong.of(l);
+    }
+
+    /**
+     * @return the total number of fields (in non-system indices), or {@link OptionalLong#empty()} if omitted (due to BwC)
+     */
+    public OptionalLong getTotalFieldCount() {
+        return ofNullable(totalFieldCount);
+    }
+
+    /**
+     * @return the total number of fields (in non-system indices) accounting for deduplication, or {@link OptionalLong#empty()} if omitted
+     * (due to BwC)
+     */
+    public OptionalLong getTotalDeduplicatedFieldCount() {
+        return ofNullable(totalDeduplicatedFieldCount);
+    }
+
+    /**
+     * @return the total size of all mappings (including those for system indices) accounting for deduplication and compression, or {@link
+     * OptionalLong#empty()} if omitted (due to BwC).
+     */
+    public OptionalLong getTotalMappingSizeBytes() {
+        return ofNullable(totalMappingSizeBytes);
+    }
+
     /**
      * Return stats about field types.
      */
@@ -197,6 +275,19 @@ public List<RuntimeFieldStats> getRuntimeFieldStats() {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("mappings");
+        if (totalFieldCount != null) {
+            builder.field("total_field_count", totalFieldCount);
+        }
+        if (totalDeduplicatedFieldCount != null) {
+            builder.field("total_deduplicated_field_count", totalDeduplicatedFieldCount);
+        }
+        if (totalMappingSizeBytes != null) {
+            builder.humanReadableField(
+                "total_deduplicated_mapping_size_in_bytes",
+                "total_deduplicated_mapping_size",
+                ByteSizeValue.ofBytes(totalMappingSizeBytes)
+            );
+        }
         builder.startArray("field_types");
         for (IndexFeatureStats st : fieldTypeStats) {
             st.toXContent(builder, params);
@@ -218,15 +309,18 @@ public String toString() {
 
     @Override
     public boolean equals(Object o) {
-        if (o instanceof MappingStats == false) {
-            return false;
-        }
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
         MappingStats that = (MappingStats) o;
-        return fieldTypeStats.equals(that.fieldTypeStats) && runtimeFieldStats.equals(that.runtimeFieldStats);
+        return Objects.equals(totalFieldCount, that.totalFieldCount)
+            && Objects.equals(totalDeduplicatedFieldCount, that.totalDeduplicatedFieldCount)
+            && Objects.equals(totalMappingSizeBytes, that.totalMappingSizeBytes)
+            && fieldTypeStats.equals(that.fieldTypeStats)
+            && runtimeFieldStats.equals(that.runtimeFieldStats);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fieldTypeStats, runtimeFieldStats);
+        return Objects.hash(totalFieldCount, totalDeduplicatedFieldCount, totalMappingSizeBytes, fieldTypeStats, runtimeFieldStats);
     }
 }

server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.hamcrest.Matchers;
 
@@ -104,6 +105,10 @@ public void testToXContent() {
         assertEquals("""
             {
               "mappings" : {
+                "total_field_count" : 12,
+                "total_deduplicated_field_count" : 6,
+                "total_deduplicated_mapping_size" : "260b",
+                "total_deduplicated_mapping_size_in_bytes" : 260,
                 "field_types" : [
                   {
                     "name" : "dense_vector",
@@ -211,6 +216,10 @@ public void testToXContentWithSomeSharedMappings() {
         assertEquals("""
             {
               "mappings" : {
+                "total_field_count" : 18,
+                "total_deduplicated_field_count" : 12,
+                "total_deduplicated_mapping_size" : "519b",
+                "total_deduplicated_mapping_size_in_bytes" : 519,
                 "field_types" : [
                   {
                     "name" : "dense_vector",
@@ -328,7 +337,7 @@ protected MappingStats createTestInstance() {
         if (randomBoolean()) {
             runtimeFieldStats.add(randomRuntimeFieldStats("long"));
         }
-        return new MappingStats(stats, runtimeFieldStats);
+        return new MappingStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), stats, runtimeFieldStats);
     }
 
     private static FieldStats randomFieldStats(String type) {
@@ -368,32 +377,41 @@ private static RuntimeFieldStats randomRuntimeFieldStats(String type) {
         return stats;
     }
 
+    @SuppressWarnings("OptionalGetWithoutIsPresent")
     @Override
     protected MappingStats mutateInstance(MappingStats instance) throws IOException {
         List<FieldStats> fieldTypes = new ArrayList<>(instance.getFieldTypeStats());
         List<RuntimeFieldStats> runtimeFieldTypes = new ArrayList<>(instance.getRuntimeFieldStats());
-        if (randomBoolean()) {
-            boolean remove = fieldTypes.size() > 0 && randomBoolean();
-            if (remove) {
-                fieldTypes.remove(randomInt(fieldTypes.size() - 1));
-            }
-            if (remove == false || randomBoolean()) {
-                FieldStats s = new FieldStats("float");
-                s.count = 13;
-                s.indexCount = 2;
-                fieldTypes.add(s);
-            }
-        } else {
-            boolean remove = runtimeFieldTypes.size() > 0 && randomBoolean();
-            if (remove) {
-                runtimeFieldTypes.remove(randomInt(runtimeFieldTypes.size() - 1));
+        long totalFieldCount = instance.getTotalFieldCount().getAsLong();
+        long totalDeduplicatedFieldCount = instance.getTotalDeduplicatedFieldCount().getAsLong();
+        long totalMappingSizeBytes = instance.getTotalMappingSizeBytes().getAsLong();
+        switch (between(1, 5)) {
+            case 1 -> {
+                boolean remove = fieldTypes.size() > 0 && randomBoolean();
+                if (remove) {
+                    fieldTypes.remove(randomInt(fieldTypes.size() - 1));
+                }
+                if (remove == false || randomBoolean()) {
+                    FieldStats s = new FieldStats("float");
+                    s.count = 13;
+                    s.indexCount = 2;
+                    fieldTypes.add(s);
+                }
             }
-            if (remove == false || randomBoolean()) {
-                runtimeFieldTypes.add(randomRuntimeFieldStats("double"));
+            case 2 -> {
+                boolean remove = runtimeFieldTypes.size() > 0 && randomBoolean();
+                if (remove) {
+                    runtimeFieldTypes.remove(randomInt(runtimeFieldTypes.size() - 1));
+                }
+                if (remove == false || randomBoolean()) {
+                    runtimeFieldTypes.add(randomRuntimeFieldStats("double"));
+                }
             }
+            case 3 -> totalFieldCount = randomValueOtherThan(totalFieldCount, ESTestCase::randomNonNegativeLong);
+            case 4 -> totalDeduplicatedFieldCount = randomValueOtherThan(totalDeduplicatedFieldCount, ESTestCase::randomNonNegativeLong);
+            case 5 -> totalMappingSizeBytes = randomValueOtherThan(totalMappingSizeBytes, ESTestCase::randomNonNegativeLong);
         }
-
-        return new MappingStats(fieldTypes, runtimeFieldTypes);
+        return new MappingStats(totalFieldCount, totalDeduplicatedFieldCount, totalMappingSizeBytes, fieldTypes, runtimeFieldTypes);
     }
 
     public void testDenseVectorType() {

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

@@ -541,6 +541,9 @@ public void testToXContent() throws IOException {
                     "file_sizes": {}
                   },
                   "mappings": {
+                    "total_field_count" : 0,
+                    "total_deduplicated_field_count" : 0,
+                    "total_deduplicated_mapping_size_in_bytes" : 0,
                     "field_types": [],
                     "runtime_field_types": []
                   },