Scripting: stats per context in nodes stats (elastic#54008)

Adds script cache stats to `_node/stats`.
If using the general cache:
```
      "script_cache": {
        "sum": {
          "compilations": 12,
          "cache_evictions": 9,
          "compilation_limit_triggered": 5
        }
      }

```
If using context caches:
```
      "script_cache": {
        "sum": {
          "compilations": 13,
          "cache_evictions": 9,
          "compilation_limit_triggered": 5
        },
        "contexts": [
          {
            "context": "aggregation_selector",
            "compilations": 8,
            "cache_evictions": 6,
            "compilation_limit_triggered": 3
          },
          {
            "context": "aggs",
            "compilations": 5,
            "cache_evictions": 3,
            "compilation_limit_triggered": 2
          },
```
Refs: elastic#50152

Author: stu-elastic

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.monitor.process.ProcessStats;
 import org.elasticsearch.node.AdaptiveSelectionStats;
+import org.elasticsearch.script.ScriptCacheStats;
 import org.elasticsearch.script.ScriptStats;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.transport.TransportStats;
@@ -82,6 +83,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private ScriptStats scriptStats;
 
+    @Nullable
+    private ScriptCacheStats scriptCacheStats;
+
     @Nullable
     private DiscoveryStats discoveryStats;
 
@@ -113,6 +117,12 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             adaptiveSelectionStats = null;
         }
+        adaptiveSelectionStats = in.readOptionalWriteable(AdaptiveSelectionStats::new);
+        if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
+            scriptCacheStats = in.readOptionalWriteable(ScriptCacheStats::new);
+        } else {
+            scriptCacheStats = null;
+        }
     }
 
     public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats indices,
@@ -122,7 +132,8 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
                      @Nullable ScriptStats scriptStats,
                      @Nullable DiscoveryStats discoveryStats,
                      @Nullable IngestStats ingestStats,
-                     @Nullable AdaptiveSelectionStats adaptiveSelectionStats) {
+                     @Nullable AdaptiveSelectionStats adaptiveSelectionStats,
+                     @Nullable ScriptCacheStats scriptCacheStats) {
         super(node);
         this.timestamp = timestamp;
         this.indices = indices;
@@ -138,6 +149,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
         this.discoveryStats = discoveryStats;
         this.ingestStats = ingestStats;
         this.adaptiveSelectionStats = adaptiveSelectionStats;
+        this.scriptCacheStats = scriptCacheStats;
     }
 
     public long getTimestamp() {
@@ -232,6 +244,11 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() {
         return adaptiveSelectionStats;
     }
 
+    @Nullable
+    public ScriptCacheStats getScriptCacheStats() {
+        return scriptCacheStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -256,6 +273,10 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_6_1_0)) {
             out.writeOptionalWriteable(adaptiveSelectionStats);
         }
+        out.writeOptionalWriteable(adaptiveSelectionStats);
+        if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
+            out.writeOptionalWriteable(scriptCacheStats);
+        }
     }
 
     @Override
@@ -319,6 +340,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getAdaptiveSelectionStats() != null) {
             getAdaptiveSelectionStats().toXContent(builder, params);
         }
+        if (getScriptCacheStats() != null) {
+            getScriptCacheStats().toXContent(builder, params);
+        }
         return builder;
     }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java

@@ -223,7 +223,8 @@ public enum Metric {
         SCRIPT("script"),
         DISCOVERY("discovery"),
         INGEST("ingest"),
-        ADAPTIVE_SELECTION("adaptive_selection");
+        ADAPTIVE_SELECTION("adaptive_selection"),
+        SCRIPT_CACHE("script_cache");
 
         private String metricName;
 

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java

@@ -149,6 +149,14 @@ public NodesStatsRequestBuilder setAdaptiveSelection(boolean adaptiveSelection)
         return this;
     }
 
+    /**
+     * Should script context cache statistics be returned
+     */
+    public NodesStatsRequestBuilder setScriptCache(boolean scriptCache) {
+        addOrRemoveMetric(scriptCache, NodesStatsRequest.Metric.SCRIPT_CACHE);
+        return this;
+    }
+
     /**
      * Helper method for adding metrics to a request
      */

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java

@@ -82,7 +82,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.SCRIPT.containedIn(metrics),
             NodesStatsRequest.Metric.DISCOVERY.containedIn(metrics),
             NodesStatsRequest.Metric.INGEST.containedIn(metrics),
-            NodesStatsRequest.Metric.ADAPTIVE_SELECTION.containedIn(metrics));
+            NodesStatsRequest.Metric.ADAPTIVE_SELECTION.containedIn(metrics),
+            NodesStatsRequest.Metric.SCRIPT_CACHE.containedIn(metrics));
     }
 
     public static class NodeStatsRequest extends BaseNodeRequest {

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -97,7 +97,7 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
         NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
         NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE,
-                true, true, true, false, true, false, false, false, false, false, true, false);
+                true, true, true, false, true, false, false, false, false, false, true, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
             for (IndexShard indexShard : indexService) {

server/src/main/java/org/elasticsearch/node/NodeService.java

@@ -103,7 +103,7 @@ public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm,
 
     public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool,
                            boolean fs, boolean transport, boolean http, boolean circuitBreaker,
-                           boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection) {
+                           boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection, boolean scriptCache) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
         return new NodeStats(transportService.getLocalNode(), System.currentTimeMillis(),
@@ -119,7 +119,8 @@ public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, bo
                 script ? scriptService.stats() : null,
                 discoveryStats ? discovery.stats() : null,
                 ingest ? ingestService.stats() : null,
-                adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null
+                adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null,
+                scriptCache ? scriptService.cacheStats() : null
         );
     }
 

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -56,14 +56,17 @@ public class ScriptCache {
     final TimeValue cacheExpire;
     final Tuple<Integer, TimeValue> rate;
     private final double compilesAllowedPerNano;
+    private final String contextRateSetting;
 
     ScriptCache(
             int cacheMaxSize,
             TimeValue cacheExpire,
-            Tuple<Integer, TimeValue> maxCompilationRate
+            Tuple<Integer, TimeValue> maxCompilationRate,
+            String contextRateSetting
     ) {
         this.cacheSize = cacheMaxSize;
         this.cacheExpire = cacheExpire;
+        this.contextRateSetting = contextRateSetting;
 
         CacheBuilder<CacheKey, Object> cacheBuilder = CacheBuilder.builder();
         if (this.cacheSize >= 0) {
@@ -175,7 +178,7 @@ void checkCompilationLimit() {
             // Otherwise reject the request
             throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" +
                 rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " +
-                "this limit can be changed by the [script.max_compilations_rate] setting",
+                "this limit can be changed by the [" + contextRateSetting + "] setting",
                 CircuitBreaker.Durability.TRANSIENT);
         }
     }

server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentFragment;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class ScriptCacheStats implements Writeable, ToXContentFragment {
+    private final Map<String,ScriptStats> context;
+    private final ScriptStats general;
+
+    public ScriptCacheStats(Map<String, ScriptStats> context) {
+        this.context = Collections.unmodifiableMap(context);
+        this.general = null;
+    }
+
+    public ScriptCacheStats(ScriptStats general) {
+        this.general = Objects.requireNonNull(general);
+        this.context = null;
+    }
+
+    public ScriptCacheStats(StreamInput in) throws IOException {
+        boolean isContext = in.readBoolean();
+        if (isContext == false) {
+            general = new ScriptStats(in);
+            context = null;
+            return;
+        }
+
+        general = null;
+        int size = in.readInt();
+        Map<String,ScriptStats> context = new HashMap<>(size);
+        for (int i=0; i < size; i++) {
+            String name = in.readString();
+            context.put(name, new ScriptStats(in));
+        }
+        this.context = Collections.unmodifiableMap(context);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (general != null) {
+            out.writeBoolean(false);
+            general.writeTo(out);
+            return;
+        }
+
+        out.writeBoolean(true);
+        out.writeInt(context.size());
+        for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
+            out.writeString(name);
+            context.get(name).writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.SCRIPT_CACHE_STATS);
+        builder.startObject(Fields.SUM);
+        if (general != null) {
+            builder.field(ScriptStats.Fields.COMPILATIONS, general.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered());
+            builder.endObject().endObject();
+            return builder;
+        }
+
+        ScriptStats sum = sum();
+        builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations());
+        builder.field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions());
+        builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered());
+        builder.endObject();
+
+        builder.startArray(Fields.CONTEXTS);
+        for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
+            ScriptStats stats = context.get(name);
+            builder.startObject();
+            builder.field(Fields.CONTEXT, name);
+            builder.field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered());
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+
+        return builder;
+    }
+
+    /**
+     * Get the context specific stats, null if using general cache
+     */
+    public Map<String, ScriptStats> getContextStats() {
+        return context;
+    }
+
+    /**
+     * Get the general stats, null if using context cache
+     */
+    public ScriptStats getGeneralStats() {
+        return general;
+    }
+
+    /**
+     * The sum of all script stats, either the general stats or the sum of all stats of the context stats.
+     */
+    public ScriptStats sum() {
+        if (general != null) {
+            return general;
+        }
+        return ScriptStats.sum(context.values());
+    }
+
+    static final class Fields {
+        static final String SCRIPT_CACHE_STATS = "script_cache";
+        static final String CONTEXT = "context";
+        static final String SUM = "sum";
+        static final String CONTEXTS = "contexts";
+    }
+}

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -568,6 +568,10 @@ public ScriptStats stats() {
         return cacheHolder.get().stats();
     }
 
+    public ScriptCacheStats cacheStats() {
+        return cacheHolder.get().cacheStats();
+    }
+
     @Override
     public void applyClusterState(ClusterChangedEvent event) {
         clusterState = event.state();
@@ -603,7 +607,8 @@ static class CacheHolder {
                     SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings),
                     compilationLimitsEnabled ?
                         SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) :
-                        SCRIPT_COMPILATION_RATE_ZERO);
+                        SCRIPT_COMPILATION_RATE_ZERO,
+                    SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey());
             }
         }
 
@@ -627,7 +632,8 @@ private static ScriptCache contextFromSettings(Settings settings, ScriptContext<
 
             return new ScriptCache(cacheSize.existsOrFallbackExists(settings) ? cacheSize.get(settings) : context.cacheSizeDefault,
                                    cacheExpire.existsOrFallbackExists(settings) ? cacheExpire.get(settings) : context.cacheExpireDefault,
-                                   compileRate);
+                                   compileRate,
+                                   SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name).getKey());
         }
 
         /**
@@ -678,6 +684,17 @@ ScriptStats stats() {
             return ScriptStats.sum(contextCache.values().stream().map(AtomicReference::get).map(ScriptCache::stats)::iterator);
         }
 
+        ScriptCacheStats cacheStats() {
+            if (general != null) {
+                return new ScriptCacheStats(general.stats());
+            }
+            Map<String, ScriptStats> context = new HashMap<>(contextCache.size());
+            for (ScriptContext<?> ctx: contexts) {
+                context.put(ctx.name, contextCache.get(ctx.name).get().stats());
+            }
+            return new ScriptCacheStats(context);
+        }
+
         /**
          * Update settings for the context cache, if we're in the context cache mode otherwise no-op.
          */