Script: Restore the scripting general cache (#79453)

Deprecate the script context cache in favor of the general cache.

Users should use the following settings:
`script.max_compilations_rate` to set the max compilation rate
  for user scripts such as filter scripts.  Certain script contexts
  that submit scripts outside of the control of the user are
  exempted from this rate limit.  Examples include runtime fields,
  ingest and watcher.

`script.cache.max_size` to set the max size of the cache.

`script.cache.expire` to set the expiration time for entries in
the cache.

Whats deprecated?
`script.max_compilations_rate: use-context`.  This special
setting value was used to turn on the script context-specific caches.

`script.context.$CONTEXT.cache_max_size`, use `script.cache.max_size`
instead.

`script.context.$CONTEXT.cache_expire`, use `script.cache.expire`
instead.

`script.context.$CONTEXT.max_compilations_rate`, use
`script.max_compilations_rate` instead.

The default cache size was increased from `100` to `3000`, which
was approximately the max cache size when using context-specific caches.

The default compilation rate limit was increased from `75/5m` to
`150/5m` to account for increasing uses of scripts.

System script contexts can now opt-out of compilation rate limiting
using a flag rather than a sentinel rate limit value.

7.16: Script: Deprecate script context cache #79508
Refs: #62899

7.16: Script: Opt-out system contexts from script compilation rate limit #79459
Refs: #62899

Author: stu-elastic

docs/reference/modules/indices/circuit_breaker.asciidoc

@@ -126,11 +126,11 @@ within a period of time.
 See the "prefer-parameters" section of the <<modules-scripting-using,scripting>>
 documentation for more information.
 
-`script.context.$CONTEXT.max_compilations_rate`::
+`script.max_compilations_rate`::
     (<<dynamic-cluster-setting,Dynamic>>)
     Limit for the number of unique dynamic scripts within a certain interval
-    that are allowed to be compiled for a given context. Defaults to `75/5m`,
-    meaning 75 every 5 minutes.
+    that are allowed to be compiled. Defaults to `150/5m`,
+    meaning 150 every 5 minutes.
 
 [[regex-circuit-breaker]]
 [discrete]

docs/reference/scripting/using.asciidoc

@@ -120,12 +120,8 @@ the `multiplier` parameter without {es} recompiling the script.
 }
 ----
 
-For most contexts, you can compile up to 75 scripts per 5 minutes by default.
-For ingest contexts, the default script compilation rate is unlimited. You
-can change these settings dynamically by setting
-`script.context.$CONTEXT.max_compilations_rate`. For example, the following
-setting limits script compilation to 100 scripts every 10 minutes for the
-{painless}/painless-field-context.html[field context]:
+You can compile up to 150 scripts per 5 minutes by default.
+For ingest contexts, the default script compilation rate is unlimited.
 
 [source,js]
 ----
@@ -406,8 +402,8 @@ small.
 
 All scripts are cached by default so that they only need to be recompiled
 when updates occur. By default, scripts do not have a time-based expiration.
-You can change this behavior by using the `script.context.$CONTEXT.cache_expire` setting.
-Use the `script.context.$CONTEXT.cache_max_size` setting to configure the size of the cache.
+You can change this behavior by using the `script.cache.expire` setting.
+Use the `script.cache.max_size` setting to configure the size of the cache.
 
 NOTE: The size of scripts is limited to 65,535 bytes. Set the value of `script.max_size_in_bytes` to increase that soft limit. If your scripts are
 really large, then consider using a

server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.monitor.os.OsStats;
 import org.elasticsearch.monitor.process.ProcessStats;
 import org.elasticsearch.node.AdaptiveSelectionStats;
+import org.elasticsearch.script.ScriptCacheStats;
 import org.elasticsearch.script.ScriptStats;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.transport.TransportStats;
@@ -71,6 +72,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private ScriptStats scriptStats;
 
+    @Nullable
+    private ScriptCacheStats scriptCacheStats;
+
     @Nullable
     private DiscoveryStats discoveryStats;
 
@@ -98,6 +102,7 @@ public NodeStats(StreamInput in) throws IOException {
         http = in.readOptionalWriteable(HttpStats::new);
         breaker = in.readOptionalWriteable(AllCircuitBreakerStats::new);
         scriptStats = in.readOptionalWriteable(ScriptStats::new);
+        scriptCacheStats = scriptStats != null ? scriptStats.toScriptCacheStats() : null;
         discoveryStats = in.readOptionalWriteable(DiscoveryStats::new);
         ingestStats = in.readOptionalWriteable(IngestStats::new);
         adaptiveSelectionStats = in.readOptionalWriteable(AdaptiveSelectionStats::new);
@@ -112,6 +117,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
                      @Nullable DiscoveryStats discoveryStats,
                      @Nullable IngestStats ingestStats,
                      @Nullable AdaptiveSelectionStats adaptiveSelectionStats,
+                     @Nullable ScriptCacheStats scriptCacheStats,
                      @Nullable IndexingPressureStats indexingPressureStats) {
         super(node);
         this.timestamp = timestamp;
@@ -128,6 +134,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats
         this.discoveryStats = discoveryStats;
         this.ingestStats = ingestStats;
         this.adaptiveSelectionStats = adaptiveSelectionStats;
+        this.scriptCacheStats = scriptCacheStats;
         this.indexingPressureStats = indexingPressureStats;
     }
 
@@ -223,6 +230,11 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() {
         return adaptiveSelectionStats;
     }
 
+    @Nullable
+    public ScriptCacheStats getScriptCacheStats() {
+        return scriptCacheStats;
+    }
+
     @Nullable
     public IndexingPressureStats getIndexingPressureStats() {
         return indexingPressureStats;
@@ -314,6 +326,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getAdaptiveSelectionStats() != null) {
             getAdaptiveSelectionStats().toXContent(builder, params);
         }
+        if (getScriptCacheStats() != null) {
+            getScriptCacheStats().toXContent(builder, params);
+        }
         if (getIndexingPressureStats() != null) {
             getIndexingPressureStats().toXContent(builder, params);
         }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -372,6 +372,9 @@ public void apply(Settings value, Settings current, Settings previous) {
             ScriptService.SCRIPT_CACHE_SIZE_SETTING,
             ScriptService.SCRIPT_CACHE_EXPIRE_SETTING,
             ScriptService.SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING,
+            ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING,
+            ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING,
+            ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING,
             ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING,
             ScriptService.SCRIPT_MAX_SIZE_IN_BYTES,
             ScriptService.TYPES_ALLOWED_SETTING,

server/src/main/java/org/elasticsearch/node/NodeService.java

@@ -118,6 +118,7 @@ public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, bo
                 discoveryStats ? coordinator.stats() : null,
                 ingest ? ingestService.stats() : null,
                 adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null,
+                scriptCache ? scriptService.cacheStats() : null,
                 indexingPressure ? this.indexingPressure.stats() : null);
     }
 

server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java

@@ -52,7 +52,7 @@ static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass) {
              * source of runaway script compilations. We think folks will
              * mostly reuse scripts though.
              */
-            ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(),
+            false,
             /*
              * Disable runtime fields scripts from being allowed
              * to be stored as part of the script meta data.

server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java

@@ -21,7 +21,7 @@ public abstract class IngestConditionalScript {
 
     /** The context used to compile {@link IngestConditionalScript} factories. */
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("processor_conditional", Factory.class,
-        200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(), true);
+        200, TimeValue.timeValueMillis(0), false, true);
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;

server/src/main/java/org/elasticsearch/script/IngestScript.java

@@ -22,7 +22,7 @@ public abstract class IngestScript {
 
     /** The context used to compile {@link IngestScript} factories. */
     public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("ingest", Factory.class,
-        200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple(), true);
+        200, TimeValue.timeValueMillis(0), false, true);
 
     /** The generic runtime parameters for the script. */
     private final Map<String, Object> params;

server/src/main/java/org/elasticsearch/script/ScriptCache.java

@@ -44,12 +44,7 @@ public class ScriptCache {
     private final double compilesAllowedPerNano;
     private final String contextRateSetting;
 
-    ScriptCache(
-            int cacheMaxSize,
-            TimeValue cacheExpire,
-            CompilationRate maxCompilationRate,
-            String contextRateSetting
-    ) {
+    ScriptCache(int cacheMaxSize, TimeValue cacheExpire, CompilationRate maxCompilationRate, String contextRateSetting) {
         this.cacheSize = cacheMaxSize;
         this.cacheExpire = cacheExpire;
         this.contextRateSetting = contextRateSetting;
@@ -94,8 +89,10 @@ <FactoryType> FactoryType compile(
                     logger.trace("context [{}]: compiling script, type: [{}], lang: [{}], options: [{}]", context.name, type,
                         lang, options);
                 }
-                // Check whether too many compilations have happened
-                checkCompilationLimit();
+                if (context.compilationRateLimited) {
+                    // Check whether too many compilations have happened
+                    checkCompilationLimit();
+                }
                 Object compiledScript = scriptEngine.compile(id, idOrCode, context, options);
                 // Since the cache key is the script content itself we don't need to
                 // invalidate/check the cache if an indexed script changes.
@@ -121,6 +118,10 @@ static <T extends Throwable> void rethrow(Throwable t) throws T {
         throw (T) t;
     }
 
+    public ScriptStats stats() {
+        return scriptMetrics.stats();
+    }
+
     public ScriptContextStats stats(String context) {
         return scriptMetrics.stats(context);
     }

server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.xcontent.ToXContentFragment;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+// This class is deprecated in favor of ScriptStats and ScriptContextStats
+public class ScriptCacheStats implements Writeable, ToXContentFragment {
+    private final Map<String, ScriptStats> context;
+    private final ScriptStats general;
+
+    public ScriptCacheStats(Map<String, ScriptStats> context) {
+        this.context = Collections.unmodifiableMap(context);
+        this.general = null;
+    }
+
+    public ScriptCacheStats(ScriptStats general) {
+        this.general = Objects.requireNonNull(general);
+        this.context = null;
+    }
+
+    public ScriptCacheStats(StreamInput in) throws IOException {
+        boolean isContext = in.readBoolean();
+        if (isContext == false) {
+            general = new ScriptStats(in);
+            context = null;
+            return;
+        }
+
+        general = null;
+        int size = in.readInt();
+        Map<String,ScriptStats> context = new HashMap<>(size);
+        for (int i=0; i < size; i++) {
+            String name = in.readString();
+            context.put(name, new ScriptStats(in));
+        }
+        this.context = Collections.unmodifiableMap(context);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (general != null) {
+            out.writeBoolean(false);
+            general.writeTo(out);
+            return;
+        }
+
+        out.writeBoolean(true);
+        out.writeInt(context.size());
+        for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
+            out.writeString(name);
+            context.get(name).writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.SCRIPT_CACHE_STATS);
+        builder.startObject(Fields.SUM);
+        if (general != null) {
+            builder.field(ScriptStats.Fields.COMPILATIONS, general.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered());
+            builder.endObject().endObject();
+            return builder;
+        }
+
+        ScriptStats sum = sum();
+        builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations());
+        builder.field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions());
+        builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered());
+        builder.endObject();
+
+        builder.startArray(Fields.CONTEXTS);
+        for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) {
+            ScriptStats stats = context.get(name);
+            builder.startObject();
+            builder.field(Fields.CONTEXT, name);
+            builder.field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations());
+            builder.field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions());
+            builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered());
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+
+        return builder;
+    }
+
+    /**
+     * Get the context specific stats, null if using general cache
+     */
+    public Map<String, ScriptStats> getContextStats() {
+        return context;
+    }
+
+    /**
+     * Get the general stats, null if using context cache
+     */
+    public ScriptStats getGeneralStats() {
+        return general;
+    }
+
+    /**
+     * The sum of all script stats, either the general stats or the sum of all stats of the context stats.
+     */
+    public ScriptStats sum() {
+        if (general != null) {
+            return general;
+        }
+        long compilations = 0;
+        long cacheEvictions = 0;
+        long compilationLimitTriggered = 0;
+        for (ScriptStats stat: context.values()) {
+            compilations += stat.getCompilations();
+            cacheEvictions += stat.getCacheEvictions();
+            compilationLimitTriggered += stat.getCompilationLimitTriggered();
+        }
+        return new ScriptStats(
+            compilations,
+            cacheEvictions,
+            compilationLimitTriggered
+        );
+    }
+
+    static final class Fields {
+        static final String SCRIPT_CACHE_STATS = "script_cache";
+        static final String CONTEXT = "context";
+        static final String SUM = "sum";
+        static final String CONTEXTS = "contexts";
+    }
+}