diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index ab0a75a007aa4..a96b6aad53f7f 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -2988,7 +2988,6 @@ - @@ -3048,7 +3047,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index dc7021e81fcaf..7b351bba55ca2 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -21,7 +21,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.settings.Setting; @@ -50,6 +49,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -320,20 +320,19 @@ public interface IndexSearcherWrapperFactory { } public IndexService newIndexService( - NodeEnvironment environment, - NamedXContentRegistry xContentRegistry, - IndexService.ShardStoreDeleter shardStoreDeleter, - CircuitBreakerService circuitBreakerService, - BigArrays bigArrays, - ThreadPool threadPool, - ScriptService scriptService, - ClusterService clusterService, - Client client, - IndicesQueryCache indicesQueryCache, - MapperRegistry mapperRegistry, - Consumer globalCheckpointSyncer, - IndicesFieldDataCache indicesFieldDataCache) - throws IOException { + NodeEnvironment environment, + NamedXContentRegistry xContentRegistry, + IndexService.ShardStoreDeleter shardStoreDeleter, + CircuitBreakerService circuitBreakerService, + BigArrays bigArrays, + ThreadPool threadPool, + ScriptService scriptService, + TemplateService templateService, + Client client, + IndicesQueryCache indicesQueryCache, + MapperRegistry mapperRegistry, + Consumer globalCheckpointSyncer, + IndicesFieldDataCache indicesFieldDataCache) throws IOException { final IndexEventListener eventListener = freeze(); IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get(); @@ -365,7 +364,7 @@ public IndexService newIndexService( } return new IndexService(indexSettings, environment, xContentRegistry, new SimilarityService(indexSettings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(), circuitBreakerService, bigArrays, threadPool, scriptService, - clusterService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry, + templateService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry, indicesFieldDataCache, globalCheckpointSyncer, searchOperationListeners, indexOperationListeners); } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index ee35993c01e79..6baf4e28df9e5 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -28,7 +28,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -68,6 +67,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; @@ -118,7 +118,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final BigArrays bigArrays; private final AsyncGlobalCheckpointTask globalCheckpointTask; private final ScriptService scriptService; - private final ClusterService clusterService; + private final TemplateService templateService; private final Client client; public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv, @@ -131,7 +131,7 @@ public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv, BigArrays bigArrays, ThreadPool threadPool, ScriptService scriptService, - ClusterService clusterService, + TemplateService templateService, Client client, QueryCache queryCache, IndexStore indexStore, @@ -158,7 +158,7 @@ public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv, this.bigArrays = bigArrays; this.threadPool = threadPool; this.scriptService = scriptService; - this.clusterService = clusterService; + this.templateService = templateService; this.client = client; this.eventListener = eventListener; this.nodeEnv = nodeEnv; @@ -473,7 +473,7 @@ public IndexSettings getIndexSettings() { public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis) { return new QueryShardContext( shardId, indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(), - similarityService(), scriptService, xContentRegistry, + similarityService(), scriptService, templateService, xContentRegistry, client, indexReader, nowInMillis); } @@ -499,6 +499,13 @@ public ScriptService getScriptService() { return scriptService; } + /** + * The {@link TemplateService} to use for this index. + */ + public TemplateService getTemplateService() { + return templateService; + } + List getIndexOperationListeners() { // pkg private for testing return indexingOperationListeners; } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 80726496a739c..c834b5b4ac272 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -25,10 +25,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import java.util.function.LongSupplier; @@ -38,6 +38,7 @@ public class QueryRewriteContext { protected final MapperService mapperService; protected final ScriptService scriptService; + private final TemplateService templateService; protected final IndexSettings indexSettings; private final NamedXContentRegistry xContentRegistry; protected final Client client; @@ -45,10 +46,11 @@ public class QueryRewriteContext { protected final LongSupplier nowInMillis; public QueryRewriteContext(IndexSettings indexSettings, MapperService mapperService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry, Client client, IndexReader reader, + TemplateService templateService, NamedXContentRegistry xContentRegistry, Client client, IndexReader reader, LongSupplier nowInMillis) { this.mapperService = mapperService; this.scriptService = scriptService; + this.templateService = templateService; this.indexSettings = indexSettings; this.xContentRegistry = xContentRegistry; this.client = client; @@ -104,7 +106,17 @@ public long nowInMillis() { } public BytesReference getTemplateBytes(Script template) { - ExecutableScript executable = scriptService.executable(template, ScriptContext.Standard.SEARCH); - return (BytesReference) executable.run(); + return templateService + .template(template.getIdOrCode(), template.getType(), + ScriptContext.Standard.SEARCH, null) + .apply(template.getParams()); + } + + public ScriptService getScriptService() { + return scriptService; + } + + public TemplateService getTemplateService() { + return templateService; } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 2b5e69947f373..0b08ccbd8ae2f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -56,6 +56,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -100,9 +101,9 @@ public String[] getTypes() { public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, - ScriptService scriptService, NamedXContentRegistry xContentRegistry, + ScriptService scriptService, TemplateService templateService, NamedXContentRegistry xContentRegistry, Client client, IndexReader reader, LongSupplier nowInMillis) { - super(indexSettings, mapperService, scriptService, xContentRegistry, client, reader, nowInMillis); + super(indexSettings, mapperService, scriptService, templateService, xContentRegistry, client, reader, nowInMillis); this.shardId = shardId; this.indexSettings = indexSettings; this.similarityService = similarityService; @@ -116,7 +117,7 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterC public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, - source.similarityService, source.scriptService, source.getXContentRegistry(), source.client, + source.similarityService, source.scriptService, source.getTemplateService(), source.getXContentRegistry(), source.client, source.reader, source.nowInMillis); this.types = source.getTypes(); } @@ -355,13 +356,14 @@ public final ExecutableScript getExecutableScript(Script script, ScriptContext c } /** - * Returns a lazily created {@link ExecutableScript} that is compiled immediately but can be pulled later once all - * parameters are available. + * Returns {@link Function} representing a script that is compiled immediately but can be pulled + * later once all parameters are available. */ - public final Function, ExecutableScript> getLazyExecutableScript(Script script, ScriptContext context) { + public final Function, ExecutableScript> getLazyExecutableScript( + Script script, ScriptContext context) { failIfFrozen(); CompiledScript executable = scriptService.compile(script, context); - return (p) -> scriptService.executable(executable, p); + return (p) -> scriptService.executable(executable, p); } /** diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index a4e4c83bc0079..10fafa898c28d 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -41,7 +41,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -110,6 +109,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchRequest; @@ -121,7 +121,6 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -162,7 +161,7 @@ public class IndicesService extends AbstractLifecycleComponent private final CircuitBreakerService circuitBreakerService; private final BigArrays bigArrays; private final ScriptService scriptService; - private final ClusterService clusterService; + private final TemplateService templateService; private final Client client; private volatile Map indices = emptyMap(); private final Map> pendingDeletes = new HashMap<>(); @@ -187,7 +186,7 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi IndexNameExpressionResolver indexNameExpressionResolver, MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry, ThreadPool threadPool, IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService, - BigArrays bigArrays, ScriptService scriptService, ClusterService clusterService, Client client, + BigArrays bigArrays, ScriptService scriptService, TemplateService templateService, Client client, MetaStateService metaStateService) { super(settings); this.threadPool = threadPool; @@ -208,7 +207,7 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi this.circuitBreakerService = circuitBreakerService; this.bigArrays = bigArrays; this.scriptService = scriptService; - this.clusterService = clusterService; + this.templateService = templateService; this.client = client; this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() { @Override @@ -447,7 +446,7 @@ private synchronized IndexService createIndexService(final String reason, bigArrays, threadPool, scriptService, - clusterService, + templateService, client, indicesQueryCache, mapperRegistry, diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java index 5249ed7a7dc84..8888ccf7cc350 100644 --- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -40,11 +40,12 @@ public class IngestService { private final PipelineStore pipelineStore; private final PipelineExecutionService pipelineExecutionService; - public IngestService(Settings settings, ThreadPool threadPool, - Environment env, ScriptService scriptService, AnalysisRegistry analysisRegistry, + public IngestService(Settings settings, ThreadPool threadPool, Environment env, + org.elasticsearch.script.TemplateService esTemplateService, + ScriptService scriptService, AnalysisRegistry analysisRegistry, List ingestPlugins) { - final TemplateService templateService = new InternalTemplateService(scriptService); + final TemplateService templateService = new InternalTemplateService(esTemplateService); Processor.Parameters parameters = new Processor.Parameters(env, scriptService, templateService, analysisRegistry, threadPool.getThreadContext()); Map processorFactories = new HashMap<>(); diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java index 26d6737706bcb..a3b51ca703c7e 100644 --- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java +++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java @@ -20,22 +20,18 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; -import java.util.Collections; import java.util.Map; +import java.util.function.Function; public class InternalTemplateService implements TemplateService { - private final ScriptService scriptService; + private final org.elasticsearch.script.TemplateService templateService; - InternalTemplateService(ScriptService scriptService) { - this.scriptService = scriptService; + InternalTemplateService(org.elasticsearch.script.TemplateService templateService) { + this.templateService = templateService; } @Override @@ -43,17 +39,12 @@ public Template compile(String template) { int mustacheStart = template.indexOf("{{"); int mustacheEnd = template.indexOf("}}"); if (mustacheStart != -1 && mustacheEnd != -1 && mustacheStart < mustacheEnd) { - Script script = new Script(ScriptType.INLINE, "mustache", template, Collections.emptyMap()); - CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST); + Function, BytesReference> compiled = templateService.template( + template, ScriptType.INLINE, ScriptContext.Standard.INGEST, null); return new Template() { @Override public String execute(Map model) { - ExecutableScript executableScript = scriptService.executable(compiledScript, model); - Object result = executableScript.run(); - if (result instanceof BytesReference) { - return ((BytesReference) result).utf8ToString(); - } - return String.valueOf(result); + return compiled.apply(model).utf8ToString(); } @Override diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java index 00e00b745a09b..dd9dc9de0eafc 100644 --- a/core/src/main/java/org/elasticsearch/node/Node.java +++ b/core/src/main/java/org/elasticsearch/node/Node.java @@ -117,6 +117,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.fetch.FetchPhase; @@ -332,9 +333,12 @@ protected Node(final Environment environment, Collection final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool, localNodeFactory::getNode); clusterService.addListener(scriptModule.getScriptService()); + clusterService.addListener(scriptModule.getTemplateService()); resourcesToClose.add(clusterService); - final IngestService ingestService = new IngestService(settings, threadPool, this.environment, - scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class)); + final IngestService ingestService = new IngestService(settings, threadPool, + this.environment, scriptModule.getTemplateService(), + scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), + pluginsService.filterPlugins(IngestPlugin.class)); final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client); ModulesBuilder modules = new ModulesBuilder(); @@ -389,12 +393,14 @@ protected Node(final Environment environment, Collection settingsModule.getClusterSettings(), analysisModule.getAnalysisRegistry(), clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry, threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptModule.getScriptService(), - clusterService, client, metaStateService); - - Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, - scriptModule.getScriptService(), xContentRegistry).stream()) - .collect(Collectors.toList()); + scriptModule.getTemplateService(), client, metaStateService); + + Collection pluginComponents = pluginsService.filterPlugins(Plugin.class) + .stream() + .flatMap(p -> p.createComponents(client, clusterService, threadPool, + resourceWatcherService, scriptModule.getScriptService(), + scriptModule.getTemplateService(), xContentRegistry).stream()) + .collect(Collectors.toList()); Collection>> customMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream() .map(Plugin::getCustomMetaDataUpgrader) @@ -439,6 +445,7 @@ protected Node(final Environment environment, Collection b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService); b.bind(BigArrays.class).toInstance(bigArrays); b.bind(ScriptService.class).toInstance(scriptModule.getScriptService()); + b.bind(TemplateService.class).toInstance(scriptModule.getTemplateService()); b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry); diff --git a/core/src/main/java/org/elasticsearch/node/NodeService.java b/core/src/main/java/org/elasticsearch/node/NodeService.java index cb245487152e3..3053d8707bbe8 100644 --- a/core/src/main/java/org/elasticsearch/node/NodeService.java +++ b/core/src/main/java/org/elasticsearch/node/NodeService.java @@ -53,7 +53,7 @@ public class NodeService extends AbstractComponent implements Closeable { private final CircuitBreakerService circuitBreakerService; private final IngestService ingestService; private final SettingsFilter settingsFilter; - private ScriptService scriptService; + private final ScriptService scriptService; private final HttpServerTransport httpServerTransport; diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java index 87c5ef9a8c694..fe207017e4b5a 100644 --- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -41,6 +41,7 @@ import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -102,10 +103,13 @@ public Collection> getGuiceServiceClasses() * @param threadPool A service to allow retrieving an executor to run an async action * @param resourceWatcherService A service to watch for changes to node local files * @param scriptService A service to allow running scripts on the local node + * @param templateService A service to render templates on the local node + * @param xContentRegistry Registry of named XContent parsers */ - public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, ScriptService scriptService, - NamedXContentRegistry xContentRegistry) { + public Collection createComponents(Client client, ClusterService clusterService, + ThreadPool threadPool, ResourceWatcherService resourceWatcherService, + ScriptService scriptService, TemplateService templateService, + NamedXContentRegistry xContentRegistry) { return Collections.emptyList(); } diff --git a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java index c1e2a43c95365..078559f919881 100644 --- a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java +++ b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java @@ -22,6 +22,7 @@ import org.elasticsearch.script.NativeScriptFactory; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.TemplateService; import java.util.Collections; import java.util.List; @@ -38,6 +39,15 @@ default ScriptEngineService getScriptEngineService(Settings settings) { return null; } + /** + * Returns a {@link TemplateService.Backend} if this plugin implements a template backend or null if it doesn't. Note that Elasticsearch + * will refuse to start if there is more than one template backend and it is bundled with Mustache. To replace that backend you'd have + * to remove the Mustache module which is super untested. + */ + default TemplateService.Backend getTemplateBackend() { + return null; + } + /** * Returns a list of {@link NativeScriptFactory} instances. */ diff --git a/core/src/main/java/org/elasticsearch/script/CachingCompiler.java b/core/src/main/java/org/elasticsearch/script/CachingCompiler.java new file mode 100644 index 0000000000000..9843b1f9d0ed4 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/CachingCompiler.java @@ -0,0 +1,386 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.cache.Cache; +import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.common.cache.RemovalListener; +import org.elasticsearch.common.cache.RemovalNotification; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.env.Environment; +import org.elasticsearch.watcher.FileChangesListener; +import org.elasticsearch.watcher.FileWatcher; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentMap; + +/** + * Manages caching, resource watching, permissions checking, and compilation of scripts (or + * templates). + */ +public abstract class CachingCompiler implements ClusterStateListener { + private static final Logger logger = ESLoggerFactory.getLogger(CachingCompiler.class); + + /** + * Compiled file scripts (or templates). Modified by the file watching process. + */ + private final ConcurrentMap fileScripts = ConcurrentCollections + .newConcurrentMap(); + + /** + * Cache of compiled dynamic scripts (or templates). + */ + private final Cache cache; + + private final Path scriptsDirectory; + + private final ScriptMetrics scriptMetrics; + + private final String type; + + private volatile ClusterState clusterState; + + public CachingCompiler(Settings settings, Environment env, + ResourceWatcherService resourceWatcherService, ScriptMetrics scriptMetrics, String type) + throws IOException { + int cacheMaxSize = ScriptService.SCRIPT_CACHE_SIZE_SETTING.get(settings); + this.scriptMetrics = scriptMetrics; + this.type = type; + + CacheBuilder cacheBuilder = CacheBuilder.builder(); + if (cacheMaxSize >= 0) { + cacheBuilder.setMaximumWeight(cacheMaxSize); + } + + TimeValue cacheExpire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.get(settings); + if (cacheExpire.getNanos() != 0) { + cacheBuilder.setExpireAfterAccess(cacheExpire); + } + + logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, + cacheExpire); + this.cache = cacheBuilder.removalListener(new CacheRemovalListener()).build(); + + // add file watcher for file scripts and templates + scriptsDirectory = env.scriptsFile(); + logger.trace("Using scripts directory [{}] ", scriptsDirectory); + FileWatcher fileWatcher = new FileWatcher(scriptsDirectory); + fileWatcher.addListener(new DirectoryChangesListener()); + if (ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.get(settings)) { + // automatic reload is enabled - register scripts + resourceWatcherService.add(fileWatcher); + } else { + // automatic reload is disabled just load scripts once + fileWatcher.init(); + } + } + + /** + * Build the cache key for a file name and its extension. Return null to indicate that the file + * type is not supported. + */ + protected abstract CacheKeyT cacheKeyForFile(String baseName, String extension); + + /** + * Build the cache key for a script (or template) stored in the cluster state. + */ + protected abstract CacheKeyT cacheKeyFromClusterState(StoredScriptSource scriptMetadata); + + /** + * Lookup a stored script (or template) from the cluster state, returning null if it is not + * found. + */ + protected abstract StoredScriptSource lookupStoredScript(ScriptMetaData scriptMetaData, + CacheKeyT cacheKey); + + /** + * Are any script contexts enabled for the given {@code cacheKey} and {@code scriptType}? Used + * to reject compilation if all script contexts are disabled and produce a nice error message + * earlier rather than later. + */ + protected abstract boolean anyScriptContextsEnabled(CacheKeyT cacheKey, + ScriptType scriptType); + + /** + * Check if a script (or template) can be executed in a particular context. + */ + protected abstract void checkContextPermissions(CacheKeyT cacheKey, ScriptType scriptType, + ScriptContext scriptContext); + + /** + * Check if too many scripts (or templates) have been compiled recently. + */ + protected abstract void checkCompilationLimit(); + + /** + * Compile an inline or stored script (or template), throwing an + * {@link IllegalArgumentException} if there is a problem. + * + * @param scriptType whether the script is inline or stored + * @param cacheKey the identifier in the cache. Note that this must contain the script's + * (or template's) source. + * @return the script (or template) compiled + */ + protected abstract CompiledScript compile(ScriptType scriptType, CacheKeyT cacheKey); + + /** + * Compile a file script (or template), throwing an {@link IllegalArgumentException} if there is + * a problem. + * + * @param cacheKey identifier in the cache as built by {@link #cacheKeyForFile(String, String)} + * @param body body of the file + * @param file path to the file for passing to the compiler in case it wants it + * @return the script (or template) compiled + */ + protected abstract CompiledScript compileFileScript(CacheKeyT cacheKey, String body, Path file); + + public final CompiledScript getScript(CacheKeyT cacheKey, ScriptType scriptType, + ScriptContext scriptContext) { + Objects.requireNonNull(cacheKey, "cacheKey is required"); + Objects.requireNonNull(scriptType, "scriptType is required"); + Objects.requireNonNull(scriptContext, "scriptContext is required"); + + // First resolve stored scripts so so we have accurate parameters for checkCanExecuteScript + if (scriptType == ScriptType.STORED) { + cacheKey = getScriptFromClusterState(cacheKey); + } + + // Validate that we can execute the script + checkContextPermissions(cacheKey, scriptType, scriptContext); + + // Lookup file scripts from the map we maintain by watching the directory + if (scriptType == ScriptType.FILE) { + CompiledScript compiled = fileScripts.get(cacheKey); + if (compiled == null) { + throw new IllegalArgumentException("unable to find file " + type + + " [" + cacheKey + "]"); + } + return compiled; + } + + // Other scripts are compiled lazily when needed so check the cache first + CompiledScript compiledScript = cache.get(cacheKey); + if (compiledScript != null) { + return compiledScript; + } + + /* Synchronize so we don't compile scripts many times during multiple shards all compiling + * a script */ + synchronized (this) { + // Double check in case it was compiled while we were waiting for the monitor + compiledScript = cache.get(cacheKey); + if (compiledScript != null) { + return compiledScript; + } + + try { + logger.trace("compiling [{}]", cacheKey); + // Check whether too many compilations have happened + checkCompilationLimit(); + compiledScript = compile(scriptType, cacheKey); + } catch (ScriptException good) { + // TODO: remove this try-catch when all script engines have good exceptions! + throw good; // its already good + } catch (Exception exception) { + throw new GeneralScriptException("Failed to compile [" + cacheKey + "]", exception); + } + scriptMetrics.onCompilation(); + cache.put(cacheKey, compiledScript); + return compiledScript; + } + } + + private CacheKeyT getScriptFromClusterState(CacheKeyT cacheKey) { + StoredScriptSource resolved = Optional.ofNullable(clusterState) + .map(ClusterState::metaData) + .map(metaData -> (ScriptMetaData) metaData.custom(ScriptMetaData.TYPE)) + .map(scriptMetaData -> lookupStoredScript(scriptMetaData, cacheKey)) + .orElseThrow(() -> new ResourceNotFoundException( + "unable to find " + type + " [" + cacheKey + "] in cluster state")); + + return cacheKeyFromClusterState(resolved); + } + + /** + * Check that a script compiles before attempting to store it. + */ + public final void checkCompileBeforeStore(StoredScriptSource source) { + CacheKeyT cacheKey = cacheKeyFromClusterState(source); + try { + if (anyScriptContextsEnabled(cacheKey, ScriptType.STORED)) { + /* Compile the script to make sure it compiles but through away the result. We'll + * populate the cache if it is ever used. */ + compile(ScriptType.STORED, cacheKey); + } else { + throw new IllegalArgumentException(type + " cannot be run under any context"); + } + } catch (ScriptException betterException) { + throw betterException; + } catch (Exception exception) { + /* Catch any less fancy scripting exceptions and send them back to the user with a + * little context information. We'd *love* to only ever through ScriptException here but + * that requires information not available to all extensions of this class. So we make + * do here. */ + throw new IllegalArgumentException("failed to compile " + type, exception); + } + } + + @Override + public final void clusterChanged(ClusterChangedEvent event) { + clusterState = event.state(); + } + + /** + * Listener to manage metrics for the script cache. + */ + private class CacheRemovalListener implements RemovalListener { + @Override + public void onRemoval(RemovalNotification notification) { + logger.debug("removed {} from cache, reason: {}", notification.getValue(), + notification.getRemovalReason()); + scriptMetrics.onCacheEviction(); + } + } + + private class DirectoryChangesListener implements FileChangesListener { + private Tuple getScriptNameExt(Path file) { + Path scriptPath = scriptsDirectory.relativize(file); + int extIndex = scriptPath.toString().lastIndexOf('.'); + if (extIndex <= 0) { + return null; + } + + String ext = scriptPath.toString().substring(extIndex + 1); + if (Strings.hasText(ext) == false) { + /* Files without extensions or with degenerate extensions like " " are not scripts + * or templates so we silently skip them rather than try to compile them. */ + return null; + } + + String scriptName = scriptPath.toString().substring(0, extIndex) + .replace(scriptPath.getFileSystem().getSeparator(), "_"); + return new Tuple<>(scriptName, ext); + } + + @Override + public void onFileInit(Path file) { + Tuple scriptNameExt = getScriptNameExt(file); + if (scriptNameExt == null) { + logger.debug("Skipped script with invalid extension : [{}]", file); + return; + } + logger.trace("Loading script file : [{}]", file); + + CacheKeyT cacheKey = cacheKeyForFile(scriptNameExt.v1(), scriptNameExt.v2()); + if (cacheKey == null) { + return; + } + try { + /* we don't know yet what the script will be used for, but if all of the operations + * for this lang with file scripts are disabled, it makes no sense to even compile + * it and cache it. */ + if (anyScriptContextsEnabled(cacheKey, ScriptType.FILE)) { + logger.info("compiling script file [{}]", file.toAbsolutePath()); + try (InputStreamReader reader = new InputStreamReader( + Files.newInputStream(file), StandardCharsets.UTF_8)) { + String body = Streams.copyToString(reader); + fileScripts.put(cacheKey, compileFileScript(cacheKey, body, file)); + scriptMetrics.onCompilation(); + } + } else { + logger.warn("skipping compile of script file [{}] as all scripted operations " + + "are disabled for file scripts", file.toAbsolutePath()); + } + } catch (ScriptException e) { + /* Attempt to extract a concise error message using the xcontent generation + * mechanisms and log that. */ + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.prettyPrint(); + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, + ToXContent.EMPTY_PARAMS, e); + builder.endObject(); + logger.warn("failed to load/compile script [{}]: {}", scriptNameExt.v1(), + builder.string()); + } catch (IOException ioe) { + ioe.addSuppressed(e); + logger.warn((Supplier) () -> new ParameterizedMessage("failed to log an " + + "appropriate warning after failing to load/compile script [{}]", + scriptNameExt.v1()), ioe); + } + /* Log at the whole exception at the debug level as well just in case the stack + * trace is important. That way you can turn on the stack trace if you need it. */ + logger.debug((Supplier) () -> new ParameterizedMessage("failed to load/compile " + + "script [{}]. full exception:", scriptNameExt.v1()), e); + } catch (Exception e) { + logger.warn((Supplier) () -> new ParameterizedMessage("failed to load/compile " + + "script [{}]", scriptNameExt.v1()), e); + } + } + + @Override + public void onFileCreated(Path file) { + onFileInit(file); + } + + @Override + public void onFileDeleted(Path file) { + Tuple scriptNameExt = getScriptNameExt(file); + if (scriptNameExt == null) { + return; + } + CacheKeyT cacheKey = cacheKeyForFile(scriptNameExt.v1(), scriptNameExt.v2()); + if (cacheKey == null) { + return; + } + logger.info("removing script file [{}]", file.toAbsolutePath()); + fileScripts.remove(cacheKey); + } + + @Override + public void onFileChanged(Path file) { + onFileInit(file); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java index 2b7feeb8d7fdf..2d1e0c1e6736a 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java @@ -57,7 +57,7 @@ public ScriptContextRegistry(Collection customScriptContex /** * @return a list that contains all the supported {@link ScriptContext}s, both standard ones and registered via plugins */ - Collection scriptContexts() { + public Collection scriptContexts() { return scriptContexts.values(); } @@ -71,10 +71,12 @@ boolean isSupportedContext(ScriptContext scriptContext) { //script contexts can be used in fine-grained settings, we need to be careful with what we allow here private void validateScriptContext(ScriptContext.Plugin scriptContext) { if (RESERVED_SCRIPT_CONTEXTS.contains(scriptContext.getPluginName())) { - throw new IllegalArgumentException("[" + scriptContext.getPluginName() + "] is a reserved name, it cannot be registered as a custom script context"); + throw new IllegalArgumentException("[" + scriptContext.getPluginName() + + "] is a reserved name, it cannot be registered as a custom script context"); } if (RESERVED_SCRIPT_CONTEXTS.contains(scriptContext.getOperation())) { - throw new IllegalArgumentException("[" + scriptContext.getOperation() + "] is a reserved name, it cannot be registered as a custom script context"); + throw new IllegalArgumentException("[" + scriptContext.getOperation() + + "] is a reserved name, it cannot be registered as a custom script context"); } } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java index f65d694aa3181..f4209523b63b2 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java @@ -19,12 +19,11 @@ package org.elasticsearch.script; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; -import org.elasticsearch.common.Strings; public class ScriptEngineRegistry { @@ -59,7 +58,7 @@ public ScriptEngineRegistry(Iterable registrations) { this.defaultInlineScriptEnableds = Collections.unmodifiableMap(inlineScriptEnableds); } - Iterable> getRegisteredScriptEngineServices() { + Collection> getRegisteredScriptEngineServices() { return registeredScriptEngineServices.keySet(); } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java index 87afc21a8c020..62137e46f532a 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java @@ -182,7 +182,7 @@ public void writeTo(StreamOutput out) throws IOException { * Convenience method to build and return a new * {@link ScriptMetaData} adding the specified stored script. */ - static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, StoredScriptSource source) { + public static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, StoredScriptSource source) { Builder builder = new Builder(previous); builder.storeScript(id, source); @@ -193,7 +193,7 @@ static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, Stored * Convenience method to build and return a new * {@link ScriptMetaData} deleting the specified stored script. */ - static ScriptMetaData deleteStoredScript(ScriptMetaData previous, String id, String lang) { + public static ScriptMetaData deleteStoredScript(ScriptMetaData previous, String id, String lang) { Builder builder = new ScriptMetaData.Builder(previous); builder.deleteScript(id, lang); @@ -425,7 +425,7 @@ public EnumSet context() { * Otherwise, returns a stored script from the deprecated namespace. Either * way an id is required. */ - StoredScriptSource getStoredScript(String id, String lang) { + public StoredScriptSource getStoredScript(String id, String lang) { if (lang == null) { return scripts.get(id); } else { diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java index 15393948d6654..ec97cbe031ef9 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModes.java @@ -47,7 +47,7 @@ public class ScriptModes { } /** - * Returns the script mode for a script of a certain written in a certain language, + * Returns the script mode for a script or template of a certain written in a certain language, * of a certain type and executing as part of a specific operation/api. * * @param lang the language that the script is written in @@ -62,7 +62,8 @@ public boolean getScriptEnabled(String lang, ScriptType scriptType, ScriptContex } Boolean scriptMode = scriptEnabled.get(getKey(lang, scriptType, scriptContext)); if (scriptMode == null) { - throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + scriptType + "], operation [" + scriptContext.getKey() + "]"); + throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + + scriptType + "], operation [" + scriptContext.getKey() + "]"); } return scriptMode; } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java index 30f84bc6960bc..a1fa53673e4b2 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -19,16 +19,17 @@ package org.elasticsearch.script; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.watcher.ResourceWatcherService; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -39,12 +40,10 @@ * Manages building {@link ScriptService} and {@link ScriptSettings} from a list of plugins. */ public class ScriptModule { - private final ScriptSettings scriptSettings; - private final ScriptService scriptService; - /** * Build from {@linkplain ScriptPlugin}s. Convenient for normal use but not great for tests. See - * {@link ScriptModule#ScriptModule(Settings, Environment, ResourceWatcherService, List, List)} for easier use in tests. + * {@link ScriptModule#ScriptModule(Settings, Environment, ResourceWatcherService, List, List, TemplateService.Backend)} + * for easier use in tests. */ public static ScriptModule create(Settings settings, Environment environment, ResourceWatcherService resourceWatcherService, List scriptPlugins) { @@ -56,23 +55,54 @@ public static ScriptModule create(Settings settings, Environment environment, scriptEngineServices.add(nativeScriptEngineService); List plugins = scriptPlugins.stream().map(x -> x.getCustomScriptContexts()).filter(Objects::nonNull) .collect(Collectors.toList()); - return new ScriptModule(settings, environment, resourceWatcherService, scriptEngineServices, plugins); + List templateBackends = scriptPlugins.stream().map(x -> x.getTemplateBackend()) + .filter(Objects::nonNull).collect(Collectors.toList()); + TemplateService.Backend templateBackend; + switch (templateBackends.size()) { + case 0: + templateBackend = null; + break; + case 1: + templateBackend = templateBackends.get(0); + break; + default: + throw new IllegalArgumentException("Elasticsearch only supports a single template backend but was started with [" + + templateBackends + "]"); + } + return new ScriptModule(settings, environment, resourceWatcherService, scriptEngineServices, plugins, templateBackend); } + private final ScriptSettings scriptSettings; + private final ScriptService scriptService; + private final TemplateService templateService; + /** * Build {@linkplain ScriptEngineService} and {@linkplain ScriptContext.Plugin}. */ public ScriptModule(Settings settings, Environment environment, ResourceWatcherService resourceWatcherService, List scriptEngineServices, - List customScriptContexts) { + List customScriptContexts, @Nullable TemplateService.Backend templateBackend) { ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineServices); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptMetrics scriptMetrics = new ScriptMetrics(); + // Note that if templateBackend is null this won't register any settings for it + scriptSettings = new ScriptSettings(scriptEngineRegistry, templateBackend, scriptContextRegistry); + try { scriptService = new ScriptService(settings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, - scriptSettings); + scriptSettings, scriptMetrics); } catch (IOException e) { - throw new RuntimeException("Couldn't setup ScriptService", e); + throw new ElasticsearchException("Couldn't setup ScriptService", e); + } + + if (templateBackend == null) { + templateBackend = new TemplatesUnsupportedBackend(); + } + try { + templateService = new TemplateService(settings, environment, resourceWatcherService, templateBackend, + scriptContextRegistry, scriptSettings, scriptMetrics); + } catch (IOException e) { + throw new ElasticsearchException("Couldn't setup TemplateService", e); } } @@ -90,10 +120,56 @@ public ScriptService getScriptService() { return scriptService; } + /** + * The service responsible for managing templates. + */ + public TemplateService getTemplateService() { + return templateService; + } + /** * Allow the script service to register any settings update handlers on the cluster settings */ public void registerClusterSettingsListeners(ClusterSettings clusterSettings) { scriptService.registerClusterSettingsListeners(clusterSettings); } + + /** + * Template backend that refuses to compile templates. Useful for starting Elasticsearch without + * the {@code lang-mustache} module. Which we do frequently during testing. Technically it'd be + * used if you started Elasticsearch without {@code lang-mustache} installed but that isn't a + * supported deployment. + */ + private static class TemplatesUnsupportedBackend implements TemplateService.Backend { + @Override + public String getType() { + throw new UnsupportedOperationException("no template backend installed"); + } + + @Override + public String getExtension() { + throw new UnsupportedOperationException("no template backend installed"); + } + + @Override + public Object compile(String scriptName, String scriptSource, Map params) { + throw new UnsupportedOperationException("no template backend installed"); + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, + Map vars) { + throw new UnsupportedOperationException("no template backend installed"); + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, + Map vars) { + throw new UnsupportedOperationException("no template backend installed"); + } + + @Override + public void close() throws IOException { + } + } } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptPermits.java b/core/src/main/java/org/elasticsearch/script/ScriptPermits.java new file mode 100644 index 0000000000000..a5efa2631f334 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/ScriptPermits.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; + +/** + * Encapsulates logic for checking if a script is allowed to be compiled. Has two kinds of checks: + *
    + *
  1. Is this script allowed to be compiled in this context? + *
  2. Have there been too many inline compilations lately? + *
+ */ +public class ScriptPermits { + private final ScriptModes scriptModes; + private final ScriptContextRegistry scriptContextRegistry; + + private int totalCompilesPerMinute; + private long lastInlineCompileTime; + private double scriptsPerMinCounter; + private double compilesAllowedPerNano; + + public ScriptPermits(Settings settings, ScriptSettings scriptSettings, + ScriptContextRegistry scriptContextRegistry) { + this.scriptModes = new ScriptModes(scriptSettings, settings); + this.scriptContextRegistry = scriptContextRegistry; + this.lastInlineCompileTime = System.nanoTime(); + this.setMaxCompilationsPerMinute( + ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.get(settings)); + } + + void registerClusterSettingsListeners(ClusterSettings clusterSettings) { + clusterSettings.addSettingsUpdateConsumer(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE, + this::setMaxCompilationsPerMinute); + } + + void setMaxCompilationsPerMinute(Integer newMaxPerMinute) { + this.totalCompilesPerMinute = newMaxPerMinute; + // Reset the counter to allow new compilations + this.scriptsPerMinCounter = totalCompilesPerMinute; + this.compilesAllowedPerNano = ((double) totalCompilesPerMinute) + / TimeValue.timeValueMinutes(1).nanos(); + } + + /** + * Check whether there have been too many compilations within the last minute, throwing a + * circuit breaking exception if so. This is a variant of the token bucket algorithm: + * https://en.wikipedia.org/wiki/Token_bucket + * + * It can be thought of as a bucket with water, every time the bucket is checked, water is added + * proportional to the amount of time that elapsed since the last time it was checked. If there + * is enough water, some is removed and the request is allowed. If there is not enough water the + * request is denied. Just like a normal bucket, if water is added that overflows the bucket, + * the extra water/capacity is discarded - there can never be more water in the bucket than the + * size of the bucket. + */ + public void checkCompilationLimit() { + long now = System.nanoTime(); + long timePassed = now - lastInlineCompileTime; + lastInlineCompileTime = now; + + scriptsPerMinCounter += (timePassed) * compilesAllowedPerNano; + + // It's been over the time limit anyway, readjust the bucket to be level + if (scriptsPerMinCounter > totalCompilesPerMinute) { + scriptsPerMinCounter = totalCompilesPerMinute; + } + + // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1 + if (scriptsPerMinCounter >= 1) { + scriptsPerMinCounter -= 1.0; + } else { + // Otherwise reject the request + throw new CircuitBreakingException( + "[script] Too many dynamic script compilations within one minute, max: [" + + totalCompilesPerMinute + "/min]; please use on-disk, indexed, or " + + "scripts with parameters instead; this limit can be changed by the [" + + ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey() + + "] setting"); + } + } + + public boolean checkContextPermissions(String lang, ScriptType scriptType, + ScriptContext scriptContext) { + assert lang != null; + if (scriptContextRegistry.isSupportedContext(scriptContext) == false) { + throw new IllegalArgumentException( + "script context [" + scriptContext.getKey() + "] not supported"); + } + return scriptModes.getScriptEnabled(lang, scriptType, scriptContext); + } +} diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java index 692e081a7ba1a..67672e615e2f9 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java @@ -19,11 +19,7 @@ package org.elasticsearch.script; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse; @@ -36,45 +32,33 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.common.cache.Cache; -import org.elasticsearch.common.cache.CacheBuilder; -import org.elasticsearch.common.cache.RemovalListener; -import org.elasticsearch.common.cache.RemovalNotification; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.env.Environment; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.watcher.FileChangesListener; -import org.elasticsearch.watcher.FileWatcher; import org.elasticsearch.watcher.ResourceWatcherService; import java.io.Closeable; import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentMap; +import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; +/** + * Source of scripts that are run in various parts of Elasticsearch. + */ public class ScriptService extends AbstractComponent implements Closeable, ClusterStateListener { static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic"; @@ -94,51 +78,40 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust private final Map scriptEnginesByLang; private final Map scriptEnginesByExt; - private final ConcurrentMap staticCache = ConcurrentCollections.newConcurrentMap(); - - private final Cache cache; - private final Path scriptsDirectory; - - private final ScriptModes scriptModes; - private final ScriptContextRegistry scriptContextRegistry; - - private final ScriptMetrics scriptMetrics = new ScriptMetrics(); - - private ClusterState clusterState; - - private int totalCompilesPerMinute; - private long lastInlineCompileTime; - private double scriptsPerMinCounter; - private double compilesAllowedPerNano; + private final ScriptMetrics scriptMetrics; + private final ScriptPermits scriptPermits; + private final CachingCompiler compiler; + private final int maxScriptSizeInBytes; + /** + * Build the service. + * + * @param settings common settings loaded at node startup + * @param env environment in which the node is running. Used to resolve the + * {@code config/scripts} directory that is scanned periodically for scripts. + * @param resourceWatcherService Scans the {@code config/scripts} directory. + * @param scriptEngineRegistry all {@link ScriptEngineService}s that we support. This delegates + * to those engines to build the actual executable. + * @param scriptContextRegistry all {@link ScriptContext}s that we support. + * @param scriptSettings settings for scripts + * @param scriptMetrics compilation metrics for scripts. This should be shared between + * {@link ScriptService} and {@link TemplateService} + * @throws IOException If there is an error scanning the {@code config/scripts} directory. + */ public ScriptService(Settings settings, Environment env, - ResourceWatcherService resourceWatcherService, ScriptEngineRegistry scriptEngineRegistry, - ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings) throws IOException { + ResourceWatcherService resourceWatcherService, + ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry, + ScriptSettings scriptSettings, ScriptMetrics scriptMetrics) throws IOException { super(settings); - Objects.requireNonNull(scriptEngineRegistry); - Objects.requireNonNull(scriptContextRegistry); - Objects.requireNonNull(scriptSettings); if (Strings.hasLength(settings.get(DISABLE_DYNAMIC_SCRIPTING_SETTING))) { - throw new IllegalArgumentException(DISABLE_DYNAMIC_SCRIPTING_SETTING + " is not a supported setting, replace with fine-grained script settings. \n" + - "Dynamic scripts can be enabled for all languages and all operations by replacing `script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml"); + throw new IllegalArgumentException(DISABLE_DYNAMIC_SCRIPTING_SETTING + " is not a supported setting, replace with " + + "fine-grained script settings. Dynamic scripts can be enabled for all languages and all operations by replacing " + + "`script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml"); } + Objects.requireNonNull(scriptEngineRegistry, "scriptEngineRegistry is required"); this.scriptEngines = scriptEngineRegistry.getRegisteredLanguages().values(); - this.scriptContextRegistry = scriptContextRegistry; - int cacheMaxSize = SCRIPT_CACHE_SIZE_SETTING.get(settings); - - CacheBuilder cacheBuilder = CacheBuilder.builder(); - if (cacheMaxSize >= 0) { - cacheBuilder.setMaximumWeight(cacheMaxSize); - } - - TimeValue cacheExpire = SCRIPT_CACHE_EXPIRE_SETTING.get(settings); - if (cacheExpire.getNanos() != 0) { - cacheBuilder.setExpireAfterAccess(cacheExpire); - } - - logger.debug("using script cache with max_size [{}], expire [{}]", cacheMaxSize, cacheExpire); - this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build(); + Objects.requireNonNull(scriptContextRegistry, "scriptContextRegistry is required"); Map enginesByLangBuilder = new HashMap<>(); Map enginesByExtBuilder = new HashMap<>(); @@ -150,323 +123,130 @@ public ScriptService(Settings settings, Environment env, this.scriptEnginesByLang = unmodifiableMap(enginesByLangBuilder); this.scriptEnginesByExt = unmodifiableMap(enginesByExtBuilder); - this.scriptModes = new ScriptModes(scriptSettings, settings); - - // add file watcher for static scripts - scriptsDirectory = env.scriptsFile(); - if (logger.isTraceEnabled()) { - logger.trace("Using scripts directory [{}] ", scriptsDirectory); - } - FileWatcher fileWatcher = new FileWatcher(scriptsDirectory); - fileWatcher.addListener(new ScriptChangesListener()); - - if (SCRIPT_AUTO_RELOAD_ENABLED_SETTING.get(settings)) { - // automatic reload is enabled - register scripts - resourceWatcherService.add(fileWatcher); - } else { - // automatic reload is disable just load scripts once - fileWatcher.init(); - } - - this.lastInlineCompileTime = System.nanoTime(); - this.setMaxCompilationsPerMinute(SCRIPT_MAX_COMPILATIONS_PER_MINUTE.get(settings)); - } - - void registerClusterSettingsListeners(ClusterSettings clusterSettings) { - clusterSettings.addSettingsUpdateConsumer(SCRIPT_MAX_COMPILATIONS_PER_MINUTE, this::setMaxCompilationsPerMinute); - } - - @Override - public void close() throws IOException { - IOUtils.close(scriptEngines); - } - - private ScriptEngineService getScriptEngineServiceForLang(String lang) { - ScriptEngineService scriptEngineService = scriptEnginesByLang.get(lang); - if (scriptEngineService == null) { - throw new IllegalArgumentException("script_lang not supported [" + lang + "]"); - } - return scriptEngineService; - } - - private ScriptEngineService getScriptEngineServiceForFileExt(String fileExtension) { - ScriptEngineService scriptEngineService = scriptEnginesByExt.get(fileExtension); - if (scriptEngineService == null) { - throw new IllegalArgumentException("script file extension not supported [" + fileExtension + "]"); - } - return scriptEngineService; - } - - void setMaxCompilationsPerMinute(Integer newMaxPerMinute) { - this.totalCompilesPerMinute = newMaxPerMinute; - // Reset the counter to allow new compilations - this.scriptsPerMinCounter = totalCompilesPerMinute; - this.compilesAllowedPerNano = ((double) totalCompilesPerMinute) / TimeValue.timeValueMinutes(1).nanos(); - } - - /** - * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script. - */ - public CompiledScript compile(Script script, ScriptContext scriptContext) { - Objects.requireNonNull(script); - Objects.requireNonNull(scriptContext); - - ScriptType type = script.getType(); - String lang = script.getLang(); - String idOrCode = script.getIdOrCode(); - Map options = script.getOptions(); - - String id = idOrCode; - - // lang may be null when looking up a stored script, so we must get the - // source to retrieve the lang before checking if the context is supported - if (type == ScriptType.STORED) { - // search template requests can possibly pass in the entire path instead - // of just an id for looking up a stored script, so we parse the path and - // check for appropriate errors - String[] path = id.split("/"); - - if (path.length == 3) { - if (lang != null && lang.equals(path[1]) == false) { - throw new IllegalStateException("conflicting script languages, found [" + path[1] + "] but expected [" + lang + "]"); + maxScriptSizeInBytes = ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.get(settings); + this.scriptMetrics = scriptMetrics; + this.scriptPermits = new ScriptPermits(settings, scriptSettings, scriptContextRegistry); + this.compiler = new CachingCompiler(settings, env, resourceWatcherService, scriptMetrics, "script") { + @Override + protected CacheKey cacheKeyForFile(String baseName, String extension) { + if (extension.equals("mustache")) { + // For backwards compatibility mustache templates are in the scripts directory and we must ignore them here + return null; } - - id = path[2]; - - deprecationLogger.deprecated("use of [" + idOrCode + "] for looking up" + - " stored scripts/templates has been deprecated, use only [" + id + "] instead"); - } else if (path.length != 1) { - throw new IllegalArgumentException("illegal stored script format [" + id + "] use only "); + ScriptEngineService engine = scriptEnginesByExt.get(extension); + if (engine == null) { + logger.warn("script file extension not supported [{}.{}]", baseName, extension); + return null; + } + return new CacheKey(engine.getType(), baseName, null); } - // a stored script must be pulled from the cluster state every time in case - // the script has been updated since the last compilation - StoredScriptSource source = getScriptFromClusterState(id, lang); - lang = source.getLang(); - idOrCode = source.getCode(); - options = source.getOptions(); - } - - // TODO: fix this through some API or something, that's wrong - // special exception to prevent expressions from compiling as update or mapping scripts - boolean expression = "expression".equals(script.getLang()); - boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()); - if (expression && notSupported) { - throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," + - " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported"); - } - - ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang); - - if (canExecuteScript(lang, type, scriptContext) == false) { - throw new IllegalStateException("scripts of type [" + script.getType() + "]," + - " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled"); - } - - if (logger.isTraceEnabled()) { - logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode); - } - - if (type == ScriptType.FILE) { - CacheKey cacheKey = new CacheKey(lang, idOrCode, options); - CompiledScript compiledScript = staticCache.get(cacheKey); - - if (compiledScript == null) { - throw new IllegalArgumentException("unable to find file script [" + idOrCode + "] using lang [" + lang + "]"); + @Override + protected CacheKey cacheKeyFromClusterState(StoredScriptSource scriptMetadata) { + return new CacheKey(scriptMetadata.getLang(), scriptMetadata.getCode(), scriptMetadata.getOptions()); } - return compiledScript; - } - - CacheKey cacheKey = new CacheKey(lang, idOrCode, options); - CompiledScript compiledScript = cache.get(cacheKey); + @Override + protected StoredScriptSource lookupStoredScript(ScriptMetaData scriptMetaData, CacheKey cacheKey) { + if (cacheKey.lang != null && isLangSupported(cacheKey.lang) == false) { + throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + cacheKey.lang + "]"); + } - if (compiledScript != null) { - return compiledScript; - } + String id = cacheKey.idOrCode; + String[] path = id.split("/"); - // Synchronize so we don't compile scripts many times during multiple shards all compiling a script - synchronized (this) { - // Retrieve it again in case it has been put by a different thread - compiledScript = cache.get(cacheKey); + if (path.length != 1) { + throw new IllegalArgumentException("illegal stored script format [" + id + "] use only "); + } - if (compiledScript == null) { - try { - // Either an un-cached inline script or indexed script - // If the script type is inline the name will be the same as the code for identification in exceptions + return scriptMetaData.getStoredScript(id, cacheKey.lang); + } - // but give the script engine the chance to be better, give it separate name + source code - // for the inline case, then its anonymous: null. - if (logger.isTraceEnabled()) { - logger.trace("compiling script, type: [{}], lang: [{}], options: [{}]", type, lang, options); + @Override + protected boolean anyScriptContextsEnabled(CacheKey cacheKey, ScriptType scriptType) { + for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { + if (scriptPermits.checkContextPermissions(cacheKey.lang, scriptType, scriptContext)) { + return true; } - // Check whether too many compilations have happened - checkCompilationLimit(); - compiledScript = new CompiledScript(type, id, lang, scriptEngineService.compile(id, idOrCode, options)); - } catch (ScriptException good) { - // TODO: remove this try-catch completely, when all script engines have good exceptions! - throw good; // its already good - } catch (Exception exception) { - throw new GeneralScriptException("Failed to compile " + type + " script [" + id + "] using lang [" + lang + "]", exception); } - - // Since the cache key is the script content itself we don't need to - // invalidate/check the cache if an indexed script changes. - scriptMetrics.onCompilation(); - cache.put(cacheKey, compiledScript); + return false; } - return compiledScript; - } - } - - /** - * Check whether there have been too many compilations within the last minute, throwing a circuit breaking exception if so. - * This is a variant of the token bucket algorithm: https://en.wikipedia.org/wiki/Token_bucket - * - * It can be thought of as a bucket with water, every time the bucket is checked, water is added proportional to the amount of time that - * elapsed since the last time it was checked. If there is enough water, some is removed and the request is allowed. If there is not - * enough water the request is denied. Just like a normal bucket, if water is added that overflows the bucket, the extra water/capacity - * is discarded - there can never be more water in the bucket than the size of the bucket. - */ - void checkCompilationLimit() { - long now = System.nanoTime(); - long timePassed = now - lastInlineCompileTime; - lastInlineCompileTime = now; - - scriptsPerMinCounter += (timePassed) * compilesAllowedPerNano; - - // It's been over the time limit anyway, readjust the bucket to be level - if (scriptsPerMinCounter > totalCompilesPerMinute) { - scriptsPerMinCounter = totalCompilesPerMinute; - } - - // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1 - if (scriptsPerMinCounter >= 1) { - scriptsPerMinCounter -= 1.0; - } else { - // Otherwise reject the request - throw new CircuitBreakingException("[script] Too many dynamic script compilations within one minute, max: [" + - totalCompilesPerMinute + "/min]; please use on-disk, indexed, or scripts with parameters instead; " + - "this limit can be changed by the [" + SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey() + "] setting"); - } - } - - public boolean isLangSupported(String lang) { - Objects.requireNonNull(lang); - - return scriptEnginesByLang.containsKey(lang); - } - - StoredScriptSource getScriptFromClusterState(String id, String lang) { - if (lang != null && isLangSupported(lang) == false) { - throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + lang + "]"); - } - - ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE); - - if (scriptMetadata == null) { - throw new ResourceNotFoundException("unable to find script [" + id + "]" + - (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state"); - } - - StoredScriptSource source = scriptMetadata.getStoredScript(id, lang); - - if (source == null) { - throw new ResourceNotFoundException("unable to find script [" + id + "]" + - (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state"); - } - - return source; - } - - public void putStoredScript(ClusterService clusterService, PutStoredScriptRequest request, - ActionListener listener) { - int max = SCRIPT_MAX_SIZE_IN_BYTES.get(settings); - - if (request.content().length() > max) { - throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + max + "] with size [" + - request.content().length() + "] for script [" + request.id() + "]"); - } - - StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType()); - - if (isLangSupported(source.getLang()) == false) { - throw new IllegalArgumentException("unable to put stored script with unsupported lang [" + source.getLang() + "]"); - } - - try { - ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(source.getLang()); - - if (isAnyScriptContextEnabled(source.getLang(), ScriptType.STORED)) { - Object compiled = scriptEngineService.compile(request.id(), source.getCode(), Collections.emptyMap()); - - if (compiled == null) { - throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" + - (source.getCode() == null ? "" : " using code [" + source.getCode() + "]")); + @Override + protected void checkContextPermissions(CacheKey cacheKey, ScriptType scriptType, ScriptContext scriptContext) { + if (isLangSupported(cacheKey.lang) == false) { + throw new IllegalArgumentException("script_lang not supported [" + cacheKey.lang + "]"); + } + // TODO: fix this through some API or something, this is a silly way to do this + // special exception to prevent expressions from compiling as update or mapping scripts + boolean expression = "expression".equals(cacheKey.lang); + boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey()); + if (expression && notSupported) { + throw new UnsupportedOperationException("scripts of type [" + scriptType + "]," + + " operation [" + scriptContext.getKey() + "] and lang [" + cacheKey.lang + "] are not supported"); + } + if (scriptPermits.checkContextPermissions(cacheKey.lang, scriptType, scriptContext) == false) { + throw new IllegalStateException("scripts of type [" + scriptType + "]," + + " operation [" + scriptContext.getKey() + "] and lang [" + cacheKey.lang + "] are disabled"); } - } else { - throw new IllegalArgumentException( - "cannot put stored script [" + request.id() + "], stored scripts cannot be run under any context"); } - } catch (ScriptException good) { - throw good; - } catch (Exception exception) { - throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]", exception); - } - clusterService.submitStateUpdateTask("put-script-" + request.id(), - new AckedClusterStateUpdateTask(request, listener) { + @Override + protected void checkCompilationLimit() { + scriptPermits.checkCompilationLimit(); + } @Override - protected PutStoredScriptResponse newResponse(boolean acknowledged) { - return new PutStoredScriptResponse(acknowledged); + protected CompiledScript compile(ScriptType scriptType, CacheKey cacheKey) { + return compile(scriptType, cacheKey, cacheKey.idOrCode, null); } @Override - public ClusterState execute(ClusterState currentState) throws Exception { - ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); - smd = ScriptMetaData.putStoredScript(smd, request.id(), source); - MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd); + protected CompiledScript compileFileScript(CacheKey cacheKey, String body, Path file) { + // pass the actual file name to the compiler (for script engines that care about this) + return compile(ScriptType.FILE, cacheKey, body, file.getFileName().toString()); + } - return ClusterState.builder(currentState).metaData(mdb).build(); + private CompiledScript compile(ScriptType scriptType, CacheKey cacheKey, String body, String fileName) { + ScriptEngineService engine = getScriptEngineServiceForLang(cacheKey.lang); + Object executable = engine.compile(fileName, body, cacheKey.options); + return new CompiledScript(scriptType, body, engine.getType(), executable); } - }); + }; } - public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request, - ActionListener listener) { - if (request.lang() != null && isLangSupported(request.lang()) == false) { - throw new IllegalArgumentException("unable to delete stored script with unsupported lang [" + request.lang() +"]"); - } + @Override + public void close() throws IOException { + IOUtils.close(scriptEngines); + } - clusterService.submitStateUpdateTask("delete-script-" + request.id(), - new AckedClusterStateUpdateTask(request, listener) { + public void registerClusterSettingsListeners(ClusterSettings clusterSettings) { + scriptPermits.registerClusterSettingsListeners(clusterSettings); + } - @Override - protected DeleteStoredScriptResponse newResponse(boolean acknowledged) { - return new DeleteStoredScriptResponse(acknowledged); - } + private ScriptEngineService getScriptEngineServiceForLang(String lang) { + ScriptEngineService scriptEngineService = scriptEnginesByLang.get(lang); + if (scriptEngineService == null) { + throw new IllegalArgumentException("script_lang not supported [" + lang + "]"); + } + return scriptEngineService; + } - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); - smd = ScriptMetaData.deleteStoredScript(smd, request.id(), request.lang()); - MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd); + /** + * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script. + */ + public CompiledScript compile(Script script, ScriptContext scriptContext) { + Objects.requireNonNull(script); + Objects.requireNonNull(scriptContext); - return ClusterState.builder(currentState).metaData(mdb).build(); - } - }); + CacheKey cacheKey = new CacheKey(script.getLang(), script.getIdOrCode(), script.getOptions()); + return compiler.getScript(cacheKey, script.getType(), scriptContext); } - public StoredScriptSource getStoredScript(ClusterState state, GetStoredScriptRequest request) { - ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE); + public boolean isLangSupported(String lang) { + Objects.requireNonNull(lang); - if (scriptMetadata != null) { - return scriptMetadata.getStoredScript(request.id(), request.lang()); - } else { - return null; - } + return scriptEnginesByLang.containsKey(lang); } /** @@ -499,141 +279,90 @@ public SearchScript search(SearchLookup lookup, CompiledScript compiledScript, return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, params); } - private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) { - for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { - if (canExecuteScript(lang, scriptType, scriptContext)) { - return true; - } - } - return false; - } - - private boolean canExecuteScript(String lang, ScriptType scriptType, ScriptContext scriptContext) { - assert lang != null; - if (scriptContextRegistry.isSupportedContext(scriptContext) == false) { - throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported"); - } - return scriptModes.getScriptEnabled(lang, scriptType, scriptContext); - } - public ScriptStats stats() { return scriptMetrics.stats(); } @Override public void clusterChanged(ClusterChangedEvent event) { - clusterState = event.state(); + compiler.clusterChanged(event); } /** - * A small listener for the script cache that calls each - * {@code ScriptEngineService}'s {@code scriptRemoved} method when the - * script has been removed from the cache + * Fetch a stored script from a cluster state. */ - private class ScriptCacheRemovalListener implements RemovalListener { - @Override - public void onRemoval(RemovalNotification notification) { - if (logger.isDebugEnabled()) { - logger.debug("removed {} from cache, reason: {}", notification.getValue(), notification.getRemovalReason()); - } - scriptMetrics.onCacheEviction(); + @Nullable + public final StoredScriptSource getStoredScript(ClusterState state, GetStoredScriptRequest request) { + ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE); + + if (scriptMetadata != null) { + return scriptMetadata.getStoredScript(request.id(), request.lang()); + } else { + return null; } } - private class ScriptChangesListener implements FileChangesListener { - - private Tuple getScriptNameExt(Path file) { - Path scriptPath = scriptsDirectory.relativize(file); - int extIndex = scriptPath.toString().lastIndexOf('.'); - if (extIndex <= 0) { - return null; - } - - String ext = scriptPath.toString().substring(extIndex + 1); - if (ext.isEmpty()) { - return null; - } - - String scriptName = scriptPath.toString().substring(0, extIndex).replace(scriptPath.getFileSystem().getSeparator(), "_"); - return new Tuple<>(scriptName, ext); + /** + * Put a stored script in the cluster state. + */ + public void putStoredScript(ClusterService clusterService, PutStoredScriptRequest request, + ActionListener listener) { + if (request.content().length() > maxScriptSizeInBytes) { + throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + maxScriptSizeInBytes + + "] with size [" + request.content().length() + "] for script [" + request.id() + "]"); } - @Override - public void onFileInit(Path file) { - Tuple scriptNameExt = getScriptNameExt(file); - if (scriptNameExt == null) { - logger.debug("Skipped script with invalid extension : [{}]", file); - return; - } - if (logger.isTraceEnabled()) { - logger.trace("Loading script file : [{}]", file); - } + StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType()); + try { + compiler.checkCompileBeforeStore(source); + } catch (IllegalArgumentException | ScriptException e) { + throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" + + (source.getCode() == null ? "" : " using code [" + source.getCode() + "]"), e); + } + clusterService.submitStateUpdateTask("put-script-" + request.id(), + new AckedClusterStateUpdateTask(request, listener) { - ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2()); - if (engineService == null) { - logger.warn("No script engine found for [{}]", scriptNameExt.v2()); - } else { - try { - //we don't know yet what the script will be used for, but if all of the operations for this lang - // with file scripts are disabled, it makes no sense to even compile it and cache it. - if (isAnyScriptContextEnabled(engineService.getType(), ScriptType.FILE)) { - logger.info("compiling script file [{}]", file.toAbsolutePath()); - try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) { - String script = Streams.copyToString(reader); - String id = scriptNameExt.v1(); - CacheKey cacheKey = new CacheKey(engineService.getType(), id, null); - // pass the actual file name to the compiler (for script engines that care about this) - Object executable = engineService.compile(file.getFileName().toString(), script, Collections.emptyMap()); - CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, id, engineService.getType(), executable); - staticCache.put(cacheKey, compiledScript); - scriptMetrics.onCompilation(); - } - } else { - logger.warn("skipping compile of script file [{}] as all scripted operations are disabled for file scripts", file.toAbsolutePath()); - } - } catch (ScriptException e) { - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.prettyPrint(); - builder.startObject(); - ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e); - builder.endObject(); - logger.warn("failed to load/compile script [{}]: {}", scriptNameExt.v1(), builder.string()); - } catch (IOException ioe) { - ioe.addSuppressed(e); - logger.warn((Supplier) () -> new ParameterizedMessage( - "failed to log an appropriate warning after failing to load/compile script [{}]", scriptNameExt.v1()), ioe); - } - /* Log at the whole exception at the debug level as well just in case the stack trace is important. That way you can - * turn on the stack trace if you need it. */ - logger.debug((Supplier) () -> new ParameterizedMessage("failed to load/compile script [{}]. full exception:", - scriptNameExt.v1()), e); - } catch (Exception e) { - logger.warn((Supplier) () -> new ParameterizedMessage("failed to load/compile script [{}]", scriptNameExt.v1()), e); + @Override + protected PutStoredScriptResponse newResponse(boolean acknowledged) { + return new PutStoredScriptResponse(acknowledged); } - } - } - @Override - public void onFileCreated(Path file) { - onFileInit(file); - } + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); + smd = ScriptMetaData.putStoredScript(smd, request.id(), source); + MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd); - @Override - public void onFileDeleted(Path file) { - Tuple scriptNameExt = getScriptNameExt(file); - if (scriptNameExt != null) { - ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2()); - assert engineService != null; - logger.info("removing script file [{}]", file.toAbsolutePath()); - staticCache.remove(new CacheKey(engineService.getType(), scriptNameExt.v1(), null)); - } - } + return ClusterState.builder(currentState).metaData(mdb).build(); + } + }); + } - @Override - public void onFileChanged(Path file) { - onFileInit(file); + /** + * Delete a stored script from the cluster state. + */ + public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request, + ActionListener listener) { + if (request.lang() != null && isLangSupported(request.lang()) == false) { + throw new IllegalArgumentException("unable to delete stored script with unsupported lang [" + request.lang() + "]"); } + clusterService.submitStateUpdateTask("delete-script-" + request.id(), + new AckedClusterStateUpdateTask(request, listener) { + @Override + protected DeleteStoredScriptResponse newResponse(boolean acknowledged) { + return new DeleteStoredScriptResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); + smd = ScriptMetaData.deleteStoredScript(smd, request.id(), request.lang()); + MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd); + + return ClusterState.builder(currentState).metaData(mdb).build(); + } + }); } private static final class CacheKey { @@ -641,10 +370,22 @@ private static final class CacheKey { final String idOrCode; final Map options; - private CacheKey(String lang, String idOrCode, Map options) { + /** + * Build the cache key. + * + * @param lang language of the script + * @param idOrCode id of the script or code for the script. For file based scripts this is + * always the id, for inline scripts this is always the code. For stored scripts this + * should be built as the id and then use + * {@link CachingCompiler#lookupStoredScript(ScriptMetaData, Object)} to replace the + * id with the source. + * @param options map of options used during script compilation. {@code null} is translated + * to {@link Collections#emptyMap()} on construction for ease of use. + */ + private CacheKey(String lang, String idOrCode, @Nullable Map options) { this.lang = lang; this.idOrCode = idOrCode; - this.options = options; + this.options = options == null ? emptyMap() : options; } @Override @@ -656,7 +397,7 @@ public boolean equals(Object o) { if (lang != null ? !lang.equals(cacheKey.lang) : cacheKey.lang != null) return false; if (!idOrCode.equals(cacheKey.idOrCode)) return false; - return options != null ? options.equals(cacheKey.options) : cacheKey.options == null; + return options.equals(cacheKey.options); } @@ -664,8 +405,18 @@ public boolean equals(Object o) { public int hashCode() { int result = lang != null ? lang.hashCode() : 0; result = 31 * result + idOrCode.hashCode(); - result = 31 * result + (options != null ? options.hashCode() : 0); + result = 31 * result + options.hashCode(); return result; } + + @Override + public String toString() { + StringBuilder result = new StringBuilder("lang=").append(lang); + result.append(", id=").append(idOrCode); + if (false == options.isEmpty()) { + result.append(", options ").append(options); + } + return result.toString(); + } } } diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java index 447097a488404..cf0d1b26523e1 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Function; public class ScriptSettings { @@ -48,11 +49,13 @@ public class ScriptSettings { private final Map> scriptContextSettingMap; private final List> scriptLanguageSettings; - public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) { + public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, TemplateService.Backend templateBackend, + ScriptContextRegistry scriptContextRegistry) { Map> scriptContextSettingMap = contextSettings(scriptContextRegistry); this.scriptContextSettingMap = Collections.unmodifiableMap(scriptContextSettingMap); - List> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry); + List> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, + scriptEngineRegistry, templateBackend, scriptContextRegistry); this.scriptLanguageSettings = Collections.unmodifiableList(scriptLanguageSettings); } @@ -68,18 +71,13 @@ private static Map> contextSettings(ScriptContex private static List> languageSettings(Map> scriptTypeSettingMap, Map> scriptContextSettingMap, ScriptEngineRegistry scriptEngineRegistry, + TemplateService.Backend templateBackend, ScriptContextRegistry scriptContextRegistry) { final List> scriptModeSettings = new ArrayList<>(); - for (final Class scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) { - if (scriptEngineService == NativeScriptEngineService.class) { - // native scripts are always enabled, and their settings can not be changed - continue; - } - final String language = scriptEngineRegistry.getLanguage(scriptEngineService); + BiConsumer populate = (language, defaultNonFileScriptMode) -> { for (final ScriptType scriptType : ScriptType.values()) { // Top level, like "script.engine.groovy.inline" - final boolean defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language); boolean defaultLangAndType = defaultNonFileScriptMode; // Files are treated differently because they are never default-deny if (ScriptType.FILE == scriptType) { @@ -141,6 +139,16 @@ private static List> languageSettings(Map c : scriptEngineRegistry.getRegisteredScriptEngineServices()) { + if (c != NativeScriptEngineService.class) { + // native scripts are always enabled, and their settings can not be changed + String language = scriptEngineRegistry.getLanguage(c); + populate.accept(language, scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language)); + } + } + if (templateBackend != null) { + populate.accept(templateBackend.getType(), true); // Templates are enabled by default } return scriptModeSettings; } diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java index 11b7821390847..c6b13baa3e311 100644 --- a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java +++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java @@ -49,7 +49,7 @@ import java.util.Objects; /** - * {@link StoredScriptSource} represents user-defined parameters for a script + * {@link StoredScriptSource} represents user-defined parameters for a script (or template) * saved in the {@link ClusterState}. */ public class StoredScriptSource extends AbstractDiffable implements Writeable, ToXContent { diff --git a/core/src/main/java/org/elasticsearch/script/TemplateService.java b/core/src/main/java/org/elasticsearch/script/TemplateService.java new file mode 100644 index 0000000000000..47bfbb7f55f0b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/script/TemplateService.java @@ -0,0 +1,240 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; + +/** + * Source of templates (think Mustache or StringTemplate). + */ +public class TemplateService implements ClusterStateListener { + /** + * Backend that implements templates. Must be provided by a module. The only tested + * implementation of this is the one provided by {@code lang-mustache} and the noop + * implementation in {@link ScriptModule}. + */ + public interface Backend extends ScriptEngineService {} // TODO customize this for templates + + private static final String DEFAULT_CONTENT_TYPE = "application/json"; + private static final Logger logger = ESLoggerFactory.getLogger(TemplateService.class); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + private final Backend backend; + private final ScriptPermits scriptPermits; + private final CachingCompiler compiler; + + /** + * Build the service. + * + * @param settings common settings loaded at node startup + * @param env environment in which the node is running. Used to resolve the + * {@code config/scripts} directory that is scanned periodically for scripts. + * @param resourceWatcherService Scans the {@code config/scripts} directory. + * @param backend the backend that actually compiles the templates. + * @param scriptSettings settings for scripts + * @param scriptMetrics compilation metrics for scripts. This should be shared between + * {@link ScriptService} and {@link TemplateService} + * @throws IOException If there is an error scanning the {@code config/scripts} directory. + */ + public TemplateService(Settings settings, Environment env, + ResourceWatcherService resourceWatcherService, Backend backend, + ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings, + ScriptMetrics scriptMetrics) throws IOException { + Objects.requireNonNull(scriptContextRegistry, "scriptContextRegistry is required"); + + this.backend = backend; + this.scriptPermits = new ScriptPermits(settings, scriptSettings, scriptContextRegistry); + this.compiler = new CachingCompiler(settings, env, resourceWatcherService, + scriptMetrics, "template") { + @Override + protected CacheKey cacheKeyForFile(String baseName, String extension) { + if (false == backend.getType().equals(extension)) { + /* For backwards compatibility templates are in the scripts directory and we + * must ignore everything but templates. */ + return null; + } + return new CacheKey(baseName, DEFAULT_CONTENT_TYPE); + } + + @Override + protected CacheKey cacheKeyFromClusterState(StoredScriptSource scriptMetadata) { + String contentType = DEFAULT_CONTENT_TYPE; + if (scriptMetadata.getOptions() != null) { + contentType = scriptMetadata.getOptions() + .getOrDefault(Script.CONTENT_TYPE_OPTION, DEFAULT_CONTENT_TYPE); + } + return new CacheKey(scriptMetadata.getCode(), contentType); + } + + @Override + protected StoredScriptSource lookupStoredScript(ScriptMetaData scriptMetaData, + CacheKey cacheKey) { + /* This process throws away cacheKey.contentType which is fine because you aren't + * allowed to specify it when using a stored template anyway. */ + String id = cacheKey.idOrCode; + // search template requests can possibly pass in the entire path instead + // of just an id for looking up a stored script, so we parse the path and + // check for appropriate errors + String[] path = id.split("/"); + + if (path.length == 3) { + id = path[2]; + deprecationLogger.deprecated("use of [{}] for looking up stored " + + "templates has been deprecated, use only [{}] instead", + cacheKey.idOrCode, id); + } else if (path.length != 1) { + throw new IllegalArgumentException( "illegal stored script format [" + id + + "] use only "); + } + + return scriptMetaData.getStoredScript(id, backend.getExtension()); + } + + @Override + protected boolean anyScriptContextsEnabled(CacheKey cacheKey, ScriptType scriptType) { + for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { + if (scriptPermits.checkContextPermissions(backend.getType(), scriptType, + scriptContext)) { + return true; + } + } + return false; + } + + @Override + protected void checkContextPermissions(CacheKey cacheKey, ScriptType scriptType, + ScriptContext scriptContext) { + if (scriptPermits.checkContextPermissions(backend.getType(), scriptType, + scriptContext) == false) { + throw new IllegalStateException("templates of [" + scriptType + "]," + + " operation [" + scriptContext.getKey() + "] are disabled"); + } + } + + @Override + protected void checkCompilationLimit() { + scriptPermits.checkCompilationLimit(); + } + + @Override + protected CompiledScript compile(ScriptType scriptType, CacheKey cacheKey) { + Object compiled = backend.compile(null, cacheKey.idOrCode, + singletonMap(Script.CONTENT_TYPE_OPTION, cacheKey.contentType)); + return new CompiledScript( + scriptType, cacheKey.idOrCode,backend.getType(), compiled); + } + + @Override + protected CompiledScript compileFileScript(CacheKey cacheKey, String body, Path file) { + Object compiled = backend.compile(file.getFileName().toString(), body, + emptyMap()); + return new CompiledScript(ScriptType.FILE, body, backend.getType(), compiled); + } + }; + } + + /** + * Lookup and/or compile a template. + * + * @param idOrCode template to look up and/or compile + * @param type whether to compile ({link ScriptType#INLINE}), lookup from cluster state + * ({@link ScriptType#STORED}), or lookup from disk ({@link ScriptType#FILE}) + * @param context context in which the template is being run + * @param contentType content type passed to the template backend during compilation + * @return the template + */ + public Function, BytesReference> template(String idOrCode, + ScriptType type, ScriptContext context, @Nullable String contentType) { + contentType = contentType == null ? DEFAULT_CONTENT_TYPE : contentType; + CacheKey cacheKey = new CacheKey(idOrCode, contentType); + CompiledScript compiled = compiler.getScript(cacheKey, type, context); + return params -> { + ExecutableScript executable = backend.executable(compiled, params); + return (BytesReference) executable.run(); + }; + } + + /** + * The language name that templates have when stored in {@link ScriptMetaData}. + */ + public String getTemplateLanguage() { + return backend.getType(); + } + + public void checkCompileBeforeStore(StoredScriptSource source) { + compiler.checkCompileBeforeStore(source); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + compiler.clusterChanged(event); + } + + private static final class CacheKey { + final String idOrCode; + final String contentType; + + private CacheKey(String idOrCode, String contentType) { + this.idOrCode = Objects.requireNonNull(idOrCode); + this.contentType = Objects.requireNonNull(contentType); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CacheKey cacheKey = (CacheKey)o; + + if (!idOrCode.equals(cacheKey.idOrCode)) return false; + return contentType.equals(cacheKey.contentType); + + } + + @Override + public int hashCode() { + int result = idOrCode.hashCode(); + result = 31 * result + contentType.hashCode(); + return result; + } + + @Override + public String toString() { + return "id=" + idOrCode + ", contentType=" + contentType; + } + } +} diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index fc862f6363864..6cdc22c5fd2f0 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -103,20 +103,19 @@ public Suggestion> innerExecute(String name, P response.addTerm(resultEntry); final BytesRefBuilder byteSpare = new BytesRefBuilder(); - final Function, ExecutableScript> collateScript = suggestion.getCollateQueryScript(); - final boolean collatePrune = (collateScript != null) && suggestion.collatePrune(); + final boolean collatePrune = + suggestion.getCollateQuery() != null && suggestion.collatePrune(); for (int i = 0; i < checkerResult.corrections.length; i++) { Correction correction = checkerResult.corrections[i]; spare.copyUTF8Bytes(correction.join(SEPARATOR, byteSpare, null, null)); boolean collateMatch = true; - if (collateScript != null) { + if (suggestion.getCollateQuery() != null) { // Checks if the template query collateScript yields any documents // from the index for a correction, collateMatch is updated final Map vars = suggestion.getCollateScriptParams(); vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString()); QueryShardContext shardContext = suggestion.getShardContext(); - final ExecutableScript executable = collateScript.apply(vars); - final BytesReference querySource = (BytesReference) executable.run(); + final BytesReference querySource = suggestion.getCollateQuery().apply(vars); try (XContentParser parser = XContentFactory.xContent(querySource).createParser(shardContext.getXContentRegistry(), querySource)) { QueryBuilder innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder(); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java index 49c5f5a9fd75f..522eb724a8bed 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; @@ -54,7 +53,6 @@ import java.util.Map.Entry; import java.util.Objects; import java.util.Set; -import java.util.function.Function; /** * Defines the actual suggest command for phrase suggestions ( phrase). @@ -630,9 +628,9 @@ public SuggestionContext build(QueryShardContext context) throws IOException { } if (this.collateQuery != null) { - Function, ExecutableScript> compiledScript = context.getLazyExecutableScript(this.collateQuery, - ScriptContext.Standard.SEARCH); - suggestionContext.setCollateQueryScript(compiledScript); + suggestionContext.setCollateQuery( + context.getTemplateService().template(collateQuery.getIdOrCode(), + collateQuery.getType(), ScriptContext.Standard.SEARCH, null)); if (this.collateParams != null) { suggestionContext.setCollateScriptParams(this.collateParams); } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 99e2e18496b61..52338ed75a4dd 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -22,9 +22,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Terms; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.script.CompiledScript; -import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.search.suggest.DirectSpellcheckerSettings; import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext; @@ -54,7 +53,7 @@ class PhraseSuggestionContext extends SuggestionContext { private boolean requireUnigram = DEFAULT_REQUIRE_UNIGRAM; private BytesRef preTag; private BytesRef postTag; - private Function, ExecutableScript> collateQueryScript; + private Function, BytesReference> collateQuery; private boolean prune = DEFAULT_COLLATE_PRUNE; private List generators = new ArrayList<>(); private Map collateScriptParams = new HashMap<>(1); @@ -194,12 +193,12 @@ public BytesRef getPostTag() { return postTag; } - Function, ExecutableScript> getCollateQueryScript() { - return collateQueryScript; + Function, BytesReference> getCollateQuery() { + return collateQuery; } - void setCollateQueryScript( Function, ExecutableScript> collateQueryScript) { - this.collateQueryScript = collateQueryScript; + void setCollateQuery(Function, BytesReference> collateQuery) { + this.collateQuery = collateQuery; } Map getCollateScriptParams() { diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 4b11697c16d9f..aef0b72a14329 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; @@ -284,9 +285,9 @@ public void testNowInScript() throws IOException { ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new MockScriptEngine("mock", scripts))); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - ScriptService scriptService = new ScriptService(baseSettings, environment, - new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); + ScriptService scriptService = new ScriptService(baseSettings, environment, new ResourceWatcherService(baseSettings, null), + scriptEngineRegistry, scriptContextRegistry, scriptSettings, new ScriptMetrics()); Settings settings = settings(Version.CURRENT).build(); UpdateHelper updateHelper = new UpdateHelper(settings, scriptService); diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 106dc9274da93..6f62984ed882b 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -34,7 +34,6 @@ import org.apache.lucene.util.SetOnce.AlreadySetException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -68,10 +67,10 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.TestSearchContext; @@ -111,7 +110,6 @@ public void addPendingDelete(ShardId shardId, IndexSettings indexSettings) { private CircuitBreakerService circuitBreakerService; private BigArrays bigArrays; private ScriptService scriptService; - private ClusterService clusterService; @Override public void setUp() throws Exception { @@ -127,10 +125,9 @@ public void setUp() throws Exception { bigArrays = new BigArrays(settings, circuitBreakerService); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(emptyList()); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); scriptService = new ScriptService(settings, environment, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry, - scriptContextRegistry, scriptSettings); - clusterService = ClusterServiceUtils.createClusterService(threadPool); + scriptContextRegistry, scriptSettings, new ScriptMetrics()); nodeEnvironment = new NodeEnvironment(settings, environment); mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry(); } @@ -138,13 +135,13 @@ public void setUp() throws Exception { @Override public void tearDown() throws Exception { super.tearDown(); - IOUtils.close(nodeEnvironment, indicesQueryCache, clusterService); + IOUtils.close(nodeEnvironment, indicesQueryCache); ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); } private IndexService newIndexService(IndexModule module) throws IOException { return module.newIndexService(nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool, - scriptService, clusterService, null, indicesQueryCache, mapperRegistry, shardId -> {}, + scriptService, null, null, indicesQueryCache, mapperRegistry, shardId -> {}, new IndicesFieldDataCache(settings, listener)); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index f6a224e8a4b3d..5c88e28617eea 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -74,7 +74,7 @@ public void modify(MappedFieldType ft) { } public void testIsFieldWithinQueryEmptyReader() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null, + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); IndexReader reader = new MultiReader(); DateFieldType ft = new DateFieldType(); @@ -85,7 +85,7 @@ public void testIsFieldWithinQueryEmptyReader() throws IOException { private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, DateTimeZone zone, DateMathParser alternateFormat) throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null, + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", randomBoolean(), randomBoolean(), null, null, context)); @@ -133,7 +133,7 @@ public void testIsFieldWithinQuery() throws IOException { DateFieldType ft2 = new DateFieldType(); ft2.setName("my_date2"); - QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null, + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context)); IOUtils.close(reader, w, dir); @@ -167,9 +167,8 @@ public void testTermQuery() { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, - new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), - indexSettings), - null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); + new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), + null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date = "2015-10-12T14:10:55"; @@ -191,7 +190,7 @@ public void testRangeQuery() throws IOException { .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), - null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); + null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date1 = "2015-10-12T14:10:55"; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index f70db120fcfc3..7d52d33016ce1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -74,7 +74,7 @@ public void testRangeQuery() throws Exception { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings); - QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), + QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type); ft.setName(FIELDNAME); diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index b83c1b2897a3c..77b7549e98aa4 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -48,7 +48,7 @@ public void testFailIfFieldMappingNotFound() { when(mapperService.getIndexSettings()).thenReturn(indexSettings); final long nowInMillis = randomNonNegativeLong(); QueryShardContext context = new QueryShardContext( - 0, indexSettings, null, null, mapperService, null, null, xContentRegistry(), null, null, + 0, indexSettings, null, null, mapperService, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); context.setAllowUnmappedFields(false); diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index d7ef534bcb2bd..284f902178716 100644 --- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -37,7 +37,7 @@ public void testRewriteMissingField() throws Exception { IndexService indexService = createIndex("test"); IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(), - null, null, xContentRegistry(), null, reader, null); + null, null, null, xContentRegistry(), null, reader, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); assertEquals(Relation.DISJOINT, range.getRelation(context)); } @@ -54,7 +54,7 @@ public void testRewriteMissingReader() throws Exception { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(), - null, null, xContentRegistry(), null, null, null); + null, null, null, xContentRegistry(), null, null, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // can't make assumptions on a missing reader, so it must return INTERSECT assertEquals(Relation.INTERSECTS, range.getRelation(context)); @@ -73,7 +73,7 @@ public void testRewriteEmptyReader() throws Exception { new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false); IndexReader reader = new MultiReader(); QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(), - null, null, xContentRegistry(), null, reader, null); + null, null, null, xContentRegistry(), null, reader, null); RangeQueryBuilder range = new RangeQueryBuilder("foo"); // no values -> DISJOINT assertEquals(Relation.DISJOINT, range.getRelation(context)); diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java index de423db442a1b..6f3cea722867d 100644 --- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java @@ -177,7 +177,7 @@ public void testQuoteFieldSuffix() { .build(); IndexMetaData indexState = IndexMetaData.builder("index").settings(indexSettings).build(); IndexSettings settings = new IndexSettings(indexState, Settings.EMPTY); - QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(), + QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, null, xContentRegistry(), null, null, System::currentTimeMillis) { @Override public MappedFieldType fieldMapper(String name) { @@ -191,7 +191,7 @@ public MappedFieldType fieldMapper(String name) { assertEquals(new TermQuery(new Term("foo.quote", "bar")), parser.parse("\"bar\"")); // Now check what happens if foo.quote does not exist - mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(), + mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, null, xContentRegistry(), null, null, System::currentTimeMillis) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 3a842a4690afa..6d5eea97207fa 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -30,7 +30,7 @@ import org.mockito.Mockito; public class IngestServiceTests extends ESTestCase { - private final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() { + private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() { @Override public Map getProcessors(Processor.Parameters parameters) { return Collections.singletonMap("foo", (factories, tag, config) -> null); @@ -39,8 +39,10 @@ public Map getProcessors(Processor.Parameters paramet public void testIngestPlugin() { ThreadPool tp = Mockito.mock(ThreadPool.class); - IngestService ingestService = new IngestService(Settings.EMPTY, tp, null, null, null, Collections.singletonList(DUMMY_PLUGIN)); - Map factories = ingestService.getPipelineStore().getProcessorFactories(); + IngestService ingestService = new IngestService(Settings.EMPTY, tp, null, null, null, null, + Collections.singletonList(DUMMY_PLUGIN)); + Map factories = ingestService.getPipelineStore() + .getProcessorFactories(); assertTrue(factories.containsKey("foo")); assertEquals(1, factories.size()); } @@ -48,8 +50,8 @@ public void testIngestPlugin() { public void testIngestPluginDuplicate() { ThreadPool tp = Mockito.mock(ThreadPool.class); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> - new IngestService(Settings.EMPTY, tp, null, null, null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN)) - ); + new IngestService(Settings.EMPTY, tp, null, null, null, null, + Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN))); assertTrue(e.getMessage(), e.getMessage().contains("already registered")); } } diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java index 92e659ac755d5..94728002f38fa 100644 --- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java @@ -46,8 +46,9 @@ ScriptService makeScriptService(Settings settings) throws Exception { MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap(scriptSource, script -> "1")); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); + return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings, + new ScriptMetrics()); } public void testFileScriptFound() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java index bf5c7e0daa782..5c9622955eb9d 100644 --- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java @@ -19,13 +19,6 @@ package org.elasticsearch.script; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -34,6 +27,13 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.watcher.ResourceWatcherService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; @@ -48,7 +48,8 @@ public void testNativeScript() throws InterruptedException { .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) .build(); ScriptModule scriptModule = new ScriptModule(settings, new Environment(settings), null, - singletonList(new NativeScriptEngineService(settings, singletonMap("my", new MyNativeScriptFactory()))), emptyList()); + singletonList(new NativeScriptEngineService(settings, singletonMap("my", new MyNativeScriptFactory()))), emptyList(), + null); List> scriptSettings = scriptModule.getSettings(); scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED); @@ -74,9 +75,9 @@ public void testFineGrainedSettingsDontAffectNativeScripts() throws IOException ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(new NativeScriptEngineService(settings, nativeScriptFactoryMap))); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(new ArrayList<>()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); ScriptService scriptService = new ScriptService(settings, environment, resourceWatcherService, scriptEngineRegistry, - scriptContextRegistry, scriptSettings); + scriptContextRegistry, scriptSettings, new ScriptMetrics()); for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) { assertThat(scriptService.compile(new Script(ScriptType.INLINE, NativeScriptEngineService.NAME, "my", Collections.emptyMap()), diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java deleted file mode 100644 index e25335e5d68e4..0000000000000 --- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.env.Environment; -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.hamcrest.Matchers.containsString; - -public class ScriptContextTests extends ESTestCase { - - private static final String PLUGIN_NAME = "testplugin"; - - ScriptService makeScriptService() throws Exception { - Settings settings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - // no file watching, so we don't need a ResourceWatcherService - .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "false") - .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false") - .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false") - .build(); - - MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); - ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); - List customContexts = Arrays.asList( - new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), - new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), - new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op")); - ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - ScriptService scriptService = new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings); - - ClusterState empty = ClusterState.builder(new ClusterName("_name")).build(); - ScriptMetaData smd = empty.metaData().custom(ScriptMetaData.TYPE); - smd = ScriptMetaData.putStoredScript(smd, "1", new StoredScriptSource(MockScriptEngine.NAME, "1", Collections.emptyMap())); - MetaData.Builder mdb = MetaData.builder(empty.getMetaData()).putCustom(ScriptMetaData.TYPE, smd); - ClusterState stored = ClusterState.builder(empty).metaData(mdb).build(); - scriptService.clusterChanged(new ClusterChangedEvent("test", stored, empty)); - - return scriptService; - } - - public void testCustomGlobalScriptContextSettings() throws Exception { - ScriptService scriptService = makeScriptService(); - for (ScriptType scriptType : ScriptType.values()) { - try { - Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap()); - scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op")); - fail("script compilation should have been rejected"); - } catch (IllegalStateException e) { - assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); - } - } - } - - public void testCustomScriptContextSettings() throws Exception { - ScriptService scriptService = makeScriptService(); - Script script = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap()); - try { - scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op")); - fail("script compilation should have been rejected"); - } catch (IllegalStateException e) { - assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled")); - } - - // still works for other script contexts - assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS)); - assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH)); - assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"))); - } - - public void testUnknownPluginScriptContext() throws Exception { - ScriptService scriptService = makeScriptService(); - for (ScriptType scriptType : ScriptType.values()) { - try { - Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap()); - scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown")); - fail("script compilation should have been rejected"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported")); - } - } - } - - public void testUnknownCustomScriptContext() throws Exception { - ScriptContext context = new ScriptContext() { - @Override - public String getKey() { - return "test"; - } - }; - ScriptService scriptService = makeScriptService(); - for (ScriptType scriptType : ScriptType.values()) { - try { - Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap()); - scriptService.compile(script, context); - fail("script compilation should have been rejected"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported")); - } - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java index f6a02ae920630..3d5c82425e2ae 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java @@ -21,24 +21,24 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptPermitsTests.MockTemplateBackend; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.common.util.set.Sets.newHashSet; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.containsString; -// TODO: this needs to be a base test class, and all scripting engines extend it public class ScriptModesTests extends ESTestCase { ScriptSettings scriptSettings; ScriptContextRegistry scriptContextRegistry; @@ -65,10 +65,11 @@ public void setupScriptEngines() { scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); scriptEngines = buildScriptEnginesByLangMap(newHashSet( //add the native engine just to make sure it gets filtered out - new NativeScriptEngineService(Settings.EMPTY, Collections.emptyMap()), + new NativeScriptEngineService(Settings.EMPTY, emptyMap()), new CustomScriptEngineService())); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines.values()); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + scriptSettings = new ScriptSettings(scriptEngineRegistry, new MockTemplateBackend(), + scriptContextRegistry); checkedSettings = new HashSet<>(); assertAllSettingsWereChecked = true; assertScriptModesNonNull = true; @@ -87,6 +88,7 @@ public void assertAllSettingsWereChecked() { assertThat(scriptModes, notNullValue()); int numberOfSettings = ScriptType.values().length * scriptContextRegistry.scriptContexts().size(); numberOfSettings += 3; // for top-level inline/store/file settings + numberOfSettings *= 2; // Once for the script engine, once for the template engine assertThat(scriptModes.scriptEnabled.size(), equalTo(numberOfSettings)); if (assertAllSettingsWereChecked) { assertThat(checkedSettings.size(), equalTo(numberOfSettings)); @@ -96,8 +98,10 @@ public void assertAllSettingsWereChecked() { public void testDefaultSettings() { this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY); - assertScriptModesAllOps(true, ScriptType.FILE); - assertScriptModesAllOps(false, ScriptType.STORED, ScriptType.INLINE); + assertScriptModesAllOps("custom", true, ScriptType.FILE); + assertScriptModesAllOps("custom", false, ScriptType.STORED, ScriptType.INLINE); + assertScriptModesAllOps("mock_template", true, ScriptType.FILE, ScriptType.STORED, + ScriptType.INLINE); } public void testMissingSetting() { @@ -130,16 +134,20 @@ public void testScriptTypeGenericSettings() { this.scriptModes = new ScriptModes(scriptSettings, builder.build()); for (int i = 0; i < randomInt; i++) { - assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]); + assertScriptModesAllOps("custom", randomScriptModes[i], randomScriptTypes[i]); + assertScriptModesAllOps("mock_template", randomScriptModes[i], randomScriptTypes[i]); } if (randomScriptTypesSet.contains(ScriptType.FILE) == false) { - assertScriptModesAllOps(true, ScriptType.FILE); + assertScriptModesAllOps("custom", true, ScriptType.FILE); + assertScriptModesAllOps("mock_template", true, ScriptType.FILE); } if (randomScriptTypesSet.contains(ScriptType.STORED) == false) { - assertScriptModesAllOps(false, ScriptType.STORED); + assertScriptModesAllOps("custom", false, ScriptType.STORED); + assertScriptModesAllOps("mock_template", true, ScriptType.STORED); } if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) { - assertScriptModesAllOps(false, ScriptType.INLINE); + assertScriptModesAllOps("custom", false, ScriptType.INLINE); + assertScriptModesAllOps("mock_template", true, ScriptType.INLINE); } } @@ -162,12 +170,14 @@ public void testScriptContextGenericSettings() { this.scriptModes = new ScriptModes(scriptSettings, builder.build()); for (int i = 0; i < randomInt; i++) { - assertScriptModesAllTypes(randomScriptModes[i], randomScriptContexts[i]); + assertScriptModesAllTypes("custom", randomScriptModes[i], randomScriptContexts[i]); + assertScriptModesAllTypes("mock_template", randomScriptModes[i], randomScriptContexts[i]); } ScriptContext[] complementOf = complementOf(randomScriptContexts); - assertScriptModes(true, new ScriptType[]{ScriptType.FILE}, complementOf); - assertScriptModes(false, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf); + assertScriptModes("custom", true, new ScriptType[] {ScriptType.FILE}, complementOf); + assertScriptModes("custom", false, new ScriptType[] {ScriptType.STORED, ScriptType.INLINE}, complementOf); + assertScriptModes("mock_template", true, ScriptType.values(), complementOf); } public void testConflictingScriptTypeAndOpGenericSettings() { @@ -178,29 +188,36 @@ public void testConflictingScriptTypeAndOpGenericSettings() { .put("script.inline", "true"); //operations generic settings have precedence over script type generic settings this.scriptModes = new ScriptModes(scriptSettings, builder.build()); - assertScriptModesAllTypes(false, scriptContext); + assertScriptModesAllTypes("custom", false, scriptContext); + assertScriptModesAllTypes("mock_template", false, scriptContext); ScriptContext[] complementOf = complementOf(scriptContext); - assertScriptModes(true, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf); - assertScriptModes(true, new ScriptType[]{ScriptType.INLINE}, complementOf); + assertScriptModes("custom", true, new ScriptType[] {ScriptType.FILE, ScriptType.STORED}, complementOf); + assertScriptModes("custom", true, new ScriptType[] {ScriptType.INLINE}, complementOf); + assertScriptModes("mock_template", true, ScriptType.values(), complementOf); } - private void assertScriptModesAllOps(boolean expectedScriptEnabled, ScriptType... scriptTypes) { - assertScriptModes(expectedScriptEnabled, scriptTypes, scriptContexts); + private void assertScriptModesAllOps(String lang, boolean expectedScriptEnabled, + ScriptType... scriptTypes) { + assertScriptModes(lang, expectedScriptEnabled, scriptTypes, scriptContexts); } - private void assertScriptModesAllTypes(boolean expectedScriptEnabled, ScriptContext... scriptContexts) { - assertScriptModes(expectedScriptEnabled, ScriptType.values(), scriptContexts); + private void assertScriptModesAllTypes(String lang, boolean expectedScriptEnabled, + ScriptContext... scriptContexts) { + assertScriptModes(lang, expectedScriptEnabled, ScriptType.values(), scriptContexts); } - private void assertScriptModes(boolean expectedScriptEnabled, ScriptType[] scriptTypes, ScriptContext... scriptContexts) { + private void assertScriptModes(String lang, boolean expectedScriptEnabled, + ScriptType[] scriptTypes, ScriptContext... scriptContexts) { assert scriptTypes.length > 0; assert scriptContexts.length > 0; for (ScriptType scriptType : scriptTypes) { - checkedSettings.add("script.engine.custom." + scriptType); + checkedSettings.add("script.engine." + lang + "." + scriptType); for (ScriptContext scriptContext : scriptContexts) { - assertThat("custom." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value", - scriptModes.getScriptEnabled("custom", scriptType, scriptContext), equalTo(expectedScriptEnabled)); - checkedSettings.add("custom." + scriptType + "." + scriptContext); + assertEquals(lang + "." + scriptType + "." + scriptContext.getKey() + + " doesn't have the expected value", + expectedScriptEnabled, + scriptModes.getScriptEnabled(lang, scriptType, scriptContext)); + checkedSettings.add(lang + "." + scriptType + "." + scriptContext); } } } diff --git a/core/src/test/java/org/elasticsearch/script/ScriptPermitsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptPermitsTests.java new file mode 100644 index 0000000000000..74b375fd2d319 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/ScriptPermitsTests.java @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; + +public class ScriptPermitsTests extends ESTestCase { + private static final String PLUGIN_NAME = "testplugin"; + + private ScriptPermits permits; + + @Before + public void makePermist() throws Exception { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + // no file watching, so we don't need a ResourceWatcherService + .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "false") + .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false") + .put("script.engine." + MockScriptEngine.NAME + ".inline." + + PLUGIN_NAME + "_custom_exp_disabled_op", "false") + .put("script.engine.mock_template.inline." + + PLUGIN_NAME + "_custom_exp_disabled_op", "false") + .build(); + + TemplateService.Backend templateBackend = new MockTemplateBackend(); + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + singletonMap("1", script -> "1")); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry( + singletonList(scriptEngine)); + List customContexts = Arrays.asList( + new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), + new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), + new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op")); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, + templateBackend, scriptContextRegistry); + permits = new ScriptPermits(settings, scriptSettings, scriptContextRegistry); + } + + public void testCompilationCircuitBreaking() throws Exception { + permits.setMaxCompilationsPerMinute(1); + permits.checkCompilationLimit(); // should pass + expectThrows(CircuitBreakingException.class, () -> permits.checkCompilationLimit()); + permits.setMaxCompilationsPerMinute(2); + permits.checkCompilationLimit(); // should pass + permits.checkCompilationLimit(); // should pass + expectThrows(CircuitBreakingException.class, () -> permits.checkCompilationLimit()); + int count = randomIntBetween(5, 50); + permits.setMaxCompilationsPerMinute(count); + for (int i = 0; i < count; i++) { + permits.checkCompilationLimit(); // should pass + } + expectThrows(CircuitBreakingException.class, () -> permits.checkCompilationLimit()); + permits.setMaxCompilationsPerMinute(0); + expectThrows(CircuitBreakingException.class, () -> permits.checkCompilationLimit()); + permits.setMaxCompilationsPerMinute(Integer.MAX_VALUE); + int largeLimit = randomIntBetween(1000, 10000); + for (int i = 0; i < largeLimit; i++) { + permits.checkCompilationLimit(); + } + } + + public void testCustomGlobalScriptContextSettings() throws Exception { + for (ScriptType scriptType : ScriptType.values()) { + assertFalse(permits.checkContextPermissions(MockScriptEngine.NAME, scriptType, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"))); + assertFalse(permits.checkContextPermissions("mock_template", scriptType, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"))); + } + } + + public void testCustomScriptContextSettings() throws Exception { + assertFalse(permits.checkContextPermissions(MockScriptEngine.NAME, ScriptType.INLINE, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"))); + assertFalse(permits.checkContextPermissions("mock_template", ScriptType.INLINE, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"))); + + // still works for other script contexts + assertTrue(permits.checkContextPermissions(MockScriptEngine.NAME, ScriptType.INLINE, + ScriptContext.Standard.AGGS)); + assertTrue(permits.checkContextPermissions(MockScriptEngine.NAME, ScriptType.INLINE, + ScriptContext.Standard.SEARCH)); + assertTrue(permits.checkContextPermissions(MockScriptEngine.NAME, ScriptType.INLINE, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"))); + assertTrue(permits.checkContextPermissions("mock_template", ScriptType.INLINE, + ScriptContext.Standard.AGGS)); + assertTrue(permits.checkContextPermissions("mock_template", ScriptType.INLINE, + ScriptContext.Standard.SEARCH)); + assertTrue(permits.checkContextPermissions("mock_template", ScriptType.INLINE, + new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"))); + } + + public void testUnknownPluginScriptContext() throws Exception { + for (ScriptType scriptType : ScriptType.values()) { + Exception e = expectThrows(IllegalArgumentException.class, () -> + permits.checkContextPermissions(MockScriptEngine.NAME, scriptType, + new ScriptContext.Plugin(PLUGIN_NAME, "unknown"))); + assertEquals("script context [testplugin_unknown] not supported", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, () -> + permits.checkContextPermissions("mock_template", scriptType, + new ScriptContext.Plugin(PLUGIN_NAME, "unknown"))); + assertEquals("script context [testplugin_unknown] not supported", e.getMessage()); + } + } + + public void testUnknownCustomScriptContext() throws Exception { + ScriptContext context = new ScriptContext() { + @Override + public String getKey() { + return "test"; + } + }; + for (ScriptType scriptType : ScriptType.values()) { + Exception e = expectThrows(IllegalArgumentException.class, () -> + permits.checkContextPermissions(MockScriptEngine.NAME, scriptType, context)); + assertEquals("script context [test] not supported", e.getMessage()); + e = expectThrows(IllegalArgumentException.class, () -> + permits.checkContextPermissions("mock_template", scriptType, context)); + assertEquals("script context [test] not supported", e.getMessage()); + } + } + + static class MockTemplateBackend implements TemplateService.Backend { + @Override + public String getType() { + return "mock_template"; + } + + @Override + public String getExtension() { + return "mock_template"; + } + + @Override + public Object compile(String scriptName, String scriptSource, Map params) { + return scriptSource; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, + Map vars) { + return new ExecutableScript() { + @Override + public void setNextVar(String name, Object value) { + } + + @Override + public Object run() { + return new BytesArray((String) compiledScript.compiled()); + } + }; + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, + Map vars) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + } + } +} diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 3482dc8bb348b..c95442171d7b0 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -18,15 +18,19 @@ */ package org.elasticsearch.script; +import com.carrotsearch.randomizedtesting.annotations.Repeat; + import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; @@ -51,7 +55,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; -//TODO: this needs to be a base test class, and all scripting engines extend it public class ScriptServiceTests extends ESTestCase { private ResourceWatcherService resourceWatcherService; @@ -67,7 +70,6 @@ public class ScriptServiceTests extends ESTestCase { private Settings baseSettings; private static final Map DEFAULT_SCRIPT_ENABLED = new HashMap<>(); - static { DEFAULT_SCRIPT_ENABLED.put(ScriptType.FILE, true); DEFAULT_SCRIPT_ENABLED.put(ScriptType.STORED, false); @@ -107,7 +109,7 @@ public void setup() throws IOException { scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineService, dangerousScriptEngineService, defaultScriptServiceEngine)); scriptContextRegistry = new ScriptContextRegistry(contexts.values()); - scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]); logger.info("--> setup script service"); scriptsFilePath = genericConfigFolder.resolve("scripts"); @@ -117,38 +119,8 @@ public void setup() throws IOException { private void buildScriptService(Settings additionalSettings) throws IOException { Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build(); Environment environment = new Environment(finalSettings); - // TODO: - scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) { - @Override - StoredScriptSource getScriptFromClusterState(String id, String lang) { - //mock the script that gets retrieved from an index - return new StoredScriptSource(lang, "100", Collections.emptyMap()); - } - }; - } - - public void testCompilationCircuitBreaking() throws Exception { - buildScriptService(Settings.EMPTY); - scriptService.setMaxCompilationsPerMinute(1); - scriptService.checkCompilationLimit(); // should pass - expectThrows(CircuitBreakingException.class, () -> scriptService.checkCompilationLimit()); - scriptService.setMaxCompilationsPerMinute(2); - scriptService.checkCompilationLimit(); // should pass - scriptService.checkCompilationLimit(); // should pass - expectThrows(CircuitBreakingException.class, () -> scriptService.checkCompilationLimit()); - int count = randomIntBetween(5, 50); - scriptService.setMaxCompilationsPerMinute(count); - for (int i = 0; i < count; i++) { - scriptService.checkCompilationLimit(); // should pass - } - expectThrows(CircuitBreakingException.class, () -> scriptService.checkCompilationLimit()); - scriptService.setMaxCompilationsPerMinute(0); - expectThrows(CircuitBreakingException.class, () -> scriptService.checkCompilationLimit()); - scriptService.setMaxCompilationsPerMinute(Integer.MAX_VALUE); - int largeLimit = randomIntBetween(1000, 10000); - for (int i = 0; i < largeLimit; i++) { - scriptService.checkCompilationLimit(); - } + scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, + scriptSettings, new ScriptMetrics()); } public void testNotSupportedDisableDynamicSetting() throws IOException { @@ -180,7 +152,7 @@ public void testScriptsWithoutExtensions() throws IOException { scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()), ScriptContext.Standard.SEARCH); fail("the script test_script should no longer exist"); } catch (IllegalArgumentException ex) { - assertThat(ex.getMessage(), containsString("unable to find file script [test_script] using lang [test]")); + assertThat(ex.getMessage(), containsString("unable to find file script [lang=test, id=test_script]")); } } @@ -219,7 +191,12 @@ public void testDefaultBehaviourFineGrainedSettings() throws IOException { builder.put("script.file", "true"); } buildScriptService(builder.build()); + + // Setup file and cluster state scripts createFileScripts("mustache", "dtest"); + ClusterState state = stateWithScripts(new Tuple<>("script", + StoredScriptSource.parse("dtest", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON))); + scriptService.clusterChanged(new ClusterChangedEvent("test", state, stateWithScripts())); for (ScriptContext scriptContext : scriptContexts) { // only file scripts are accepted by default @@ -291,6 +268,9 @@ public void testFineGrainedSettings() throws IOException { buildScriptService(builder.build()); createFileScripts("expression", "mustache", "dtest"); + ClusterState state = stateWithScripts(new Tuple<>("script", StoredScriptSource.parse(dangerousScriptEngineService.getType(), + new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON))); + scriptService.clusterChanged(new ClusterChangedEvent("test", state, stateWithScripts())); for (ScriptType scriptType : ScriptType.values()) { //make sure file scripts have a different name than inline ones. @@ -321,6 +301,7 @@ public void testFineGrainedSettings() throws IOException { public void testCompileNonRegisteredContext() throws IOException { buildScriptService(Settings.EMPTY); + String pluginName; String unknownContext; do { @@ -330,8 +311,13 @@ public void testCompileNonRegisteredContext() throws IOException { String type = scriptEngineService.getType(); try { - scriptService.compile(new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()), - new ScriptContext.Plugin(pluginName, unknownContext)); + Script script = new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()); + if (script.getType() == ScriptType.STORED) { + ClusterState state = stateWithScripts(new Tuple<>(script.getIdOrCode(), + StoredScriptSource.parse(script.getLang(), new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON))); + scriptService.clusterChanged(new ClusterChangedEvent("test", state, stateWithScripts())); + } + scriptService.compile(script, new ScriptContext.Plugin(pluginName, unknownContext)); fail("script compilation should have been rejected"); } catch(IllegalArgumentException e) { assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported")); @@ -385,6 +371,10 @@ public void testFileScriptCountedInCompilationStats() throws IOException { public void testIndexedScriptCountedInCompilationStats() throws IOException { buildScriptService(Settings.EMPTY); + ClusterState cs = stateWithScripts( + new Tuple<>("script", StoredScriptSource.parse("test", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON))); + scriptService.clusterChanged(new ClusterChangedEvent("test", cs, stateWithScripts())); + scriptService.compile(new Script(ScriptType.STORED, "test", "script", Collections.emptyMap()), randomFrom(scriptContexts)); assertEquals(1L, scriptService.stats().getCompilations()); } @@ -436,12 +426,8 @@ public void testDeleteScript() throws Exception { public void testGetStoredScript() throws Exception { buildScriptService(Settings.EMPTY); - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metaData(MetaData.builder() - .putCustom(ScriptMetaData.TYPE, - new ScriptMetaData.Builder(null).storeScript("_id", - StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON)).build())) - .build(); + ClusterState cs = stateWithScripts( + new Tuple<>("_id", StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON))); assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")).getCode()); assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id2", "_lang"))); @@ -450,6 +436,18 @@ public void testGetStoredScript() throws Exception { assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang"))); } + @SafeVarargs + private final ClusterState stateWithScripts(Tuple... scripts) { + ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null); + for (Tuple script : scripts) { + builder.storeScript(script.v1(), script.v2()); + } + return ClusterState.builder(new ClusterName(getTestName())) + .metaData(MetaData.builder() + .putCustom(ScriptMetaData.TYPE, builder.build())) + .build(); + } + private void createFileScripts(String... langs) throws IOException { for (String lang : langs) { Path scriptPath = scriptsFilePath.resolve("file_script." + lang); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java index abda0376a2cfa..a8e4699945271 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java @@ -25,26 +25,32 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; -import java.util.Collections; import java.util.Iterator; import java.util.Map; -import static org.hamcrest.Matchers.containsString; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class ScriptSettingsTests extends ESTestCase { - public void testSettingsAreProperlyPropogated() { - ScriptEngineRegistry scriptEngineRegistry = - new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService())); - ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry( + singletonList(new CustomScriptEngineService())); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(emptyList()); + TemplateService.Backend templateBackend = mock(TemplateService.Backend.class); + when(templateBackend.getType()).thenReturn("test_template_backend"); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, templateBackend, + scriptContextRegistry); boolean enabled = randomBoolean(); Settings s = Settings.builder().put("script.inline", enabled).build(); - for (Iterator> iter = scriptSettings.getScriptLanguageSettings().iterator(); iter.hasNext();) { + for (Iterator> iter = scriptSettings.getScriptLanguageSettings() + .iterator(); iter.hasNext();) { Setting setting = iter.next(); if (setting.getKey().endsWith(".inline")) { - assertThat("inline settings should have propagated", setting.get(s), equalTo(enabled)); + assertThat("inline settings should have propagated", setting.get(s), + equalTo(enabled)); assertThat(setting.getDefaultRaw(s), equalTo(Boolean.toString(enabled))); } } @@ -70,12 +76,14 @@ public Object compile(String scriptName, String scriptSource, Map vars) { + public ExecutableScript executable(CompiledScript compiledScript, + @Nullable Map vars) { return null; } @Override - public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map vars) { + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, + @Nullable Map vars) { return null; } diff --git a/core/src/test/java/org/elasticsearch/script/TemplateServiceTests.java b/core/src/test/java/org/elasticsearch/script/TemplateServiceTests.java new file mode 100644 index 0000000000000..05311fae427be --- /dev/null +++ b/core/src/test/java/org/elasticsearch/script/TemplateServiceTests.java @@ -0,0 +1,200 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + +public class TemplateServiceTests extends ESTestCase { + private ResourceWatcherService resourceWatcherService; + private ScriptContextRegistry scriptContextRegistry; + private ScriptSettings scriptSettings; + private Path fileTemplatesPath; + private Settings settings; + private TemplateService.Backend backend; + private TemplateService templateService; + + private static final Map DEFAULT_SCRIPT_ENABLED = new HashMap<>(); + static { + DEFAULT_SCRIPT_ENABLED.put(ScriptType.FILE, true); + DEFAULT_SCRIPT_ENABLED.put(ScriptType.STORED, false); + DEFAULT_SCRIPT_ENABLED.put(ScriptType.INLINE, false); + } + + @Before + public void setup() throws IOException { + Path genericConfigFolder = createTempDir(); + settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder) + .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 10000) + .build(); + resourceWatcherService = new ResourceWatcherService(settings, null); + //randomly register custom script contexts + int randomInt = randomIntBetween(0, 3); + //prevent duplicates using map + Map contexts = new HashMap<>(); + for (int i = 0; i < randomInt; i++) { + String plugin; + do { + plugin = randomAsciiOfLength(randomIntBetween(1, 10)); + } while (ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS.contains(plugin)); + String operation; + do { + operation = randomAsciiOfLength(randomIntBetween(1, 30)); + } while (ScriptContextRegistry.RESERVED_SCRIPT_CONTEXTS.contains(operation)); + String context = plugin + "_" + operation; + contexts.put(context, new ScriptContext.Plugin(plugin, operation)); + } + scriptContextRegistry = new ScriptContextRegistry(contexts.values()); + backend = new DummyBackend(); + scriptSettings = new ScriptSettings(new ScriptEngineRegistry(emptyList()), backend, + scriptContextRegistry); + logger.info("--> setup script service"); + fileTemplatesPath = genericConfigFolder.resolve("scripts"); + Files.createDirectories(fileTemplatesPath); + Environment environment = new Environment(settings); + templateService = new TemplateService(settings, environment, resourceWatcherService, + backend, scriptContextRegistry, scriptSettings, new ScriptMetrics()); + } + + public void testFileTemplates() throws IOException { + String body = "{\"test\":\"test\"}"; + Path testFileWithExt = fileTemplatesPath.resolve("test.test_template_backend"); + Streams.copy(body.getBytes("UTF-8"), Files.newOutputStream(testFileWithExt)); + resourceWatcherService.notifyNow(); + + assertEquals(new BytesArray(body), templateService.template("test", ScriptType.FILE, + ScriptContext.Standard.SEARCH, null).apply(emptyMap())); + + Files.delete(testFileWithExt); + resourceWatcherService.notifyNow(); + + Exception e = expectThrows(IllegalArgumentException.class, () -> templateService + .template("test", ScriptType.FILE, ScriptContext.Standard.SEARCH, null)); + assertEquals("unable to find file template [id=test, contentType=text/plain]", + e.getMessage()); + } + + public void testStoredTemplates() throws IOException { + String body = "{\"test\":\"test\"}"; + + ClusterState newState = ClusterState.builder(new ClusterName("test")) + .metaData(MetaData.builder() + .putCustom(ScriptMetaData.TYPE, new ScriptMetaData.Builder(null) + .storeScript("test_template_backend#test", new StoredScriptSource( + "test_template_backend", body, emptyMap())) + .build()) + .build()) + .build(); + templateService.clusterChanged( + new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE)); + + assertEquals(new BytesArray(body), templateService + .template("test", ScriptType.STORED, ScriptContext.Standard.SEARCH, null) + .apply(emptyMap())); + + ClusterState oldState = newState; + newState = ClusterState.builder(oldState) + .metaData(MetaData.builder(oldState.metaData()) + .putCustom(ScriptMetaData.TYPE, new ScriptMetaData.Builder(null).build()) + .build()) + .build(); + templateService.clusterChanged(new ClusterChangedEvent("test", newState, oldState)); + + Exception e = expectThrows(ResourceNotFoundException.class, () -> templateService + .template("test", ScriptType.STORED, ScriptContext.Standard.SEARCH, null)); + assertEquals("unable to find template [id=test, contentType=text/plain] in cluster state", + e.getMessage()); + } + + public void testInlineTemplates() throws IOException { + String body = "{\"test\":\"test\"}"; + + assertEquals(new BytesArray(body), templateService + .template(body, ScriptType.INLINE, ScriptContext.Standard.SEARCH, null) + .apply(emptyMap())); + } + + /** + * Dummy backend that just returns the script's source when run. + */ + private static class DummyBackend implements TemplateService.Backend { + @Override + public String getType() { + return "test_template_backend"; + } + + @Override + public String getExtension() { + return "test_template_backend"; + } + + @Override + public Object compile(String scriptName, String scriptSource, Map params) { + return scriptSource; + } + + @Override + public ExecutableScript executable(CompiledScript compiledScript, + Map vars) { + return new ExecutableScript() { + @Override + public void setNextVar(String name, Object value) { + } + + @Override + public Object run() { + return new BytesArray((String) compiledScript.compiled()); + } + }; + } + + @Override + public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, + Map vars) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + } + } +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java index 16ba3ed6958b7..583406e226ac5 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java @@ -99,7 +99,7 @@ public void testParseAndValidate() { SearchContext context = mock(SearchContext.class); QueryShardContext qsc = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null, - null, xContentRegistry(), null, null, () -> now); + null, null, xContentRegistry(), null, null, () -> now); when(context.getQueryShardContext()).thenReturn(qsc); FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime"); DocValueFormat format = new DocValueFormat.DateTime(formatter, DateTimeZone.UTC); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java index b82e822f6b04c..9daf89bbbde4b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetricTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; @@ -80,10 +81,10 @@ protected ScriptService mockScriptService() { })); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); try { return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, - scriptSettings); + scriptSettings, new ScriptMetrics()); } catch (IOException e) { throw new ElasticsearchException(e); } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index d9eb76310d241..6a8c18c69ff63 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.ScriptType; @@ -158,15 +159,15 @@ protected QueryShardContext queryShardContextMock(final MappedFieldType[] fieldT MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, SCRIPTS); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(scriptEngine)); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); ScriptService scriptService; try { - scriptService = new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, - scriptSettings); + scriptService = new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, + scriptSettings, new ScriptMetrics()); } catch (IOException e) { throw new ElasticsearchException(e); } - return new QueryShardContext(0, idxSettings, null, null, null, null, scriptService, xContentRegistry(), + return new QueryShardContext(0, idxSettings, null, null, null, null, scriptService, null, xContentRegistry(), null, null, System::currentTimeMillis); } } diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index e33b201bf2252..f06ecbb87da2f 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -265,7 +265,7 @@ public void testBuildSearchContextHighlight() throws IOException { Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), + QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, null, xContentRegistry(), null, null, System::currentTimeMillis) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index d0132cca7ad50..2257276ec856c 100644 --- a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -200,7 +200,7 @@ public void testSerialize50Request() throws IOException { IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); QueryShardContext context = new QueryShardContext( - 0, indexSettings, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); + 0, indexSettings, null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); readRequest.rewrite(context); QueryBuilder queryBuilder = readRequest.filteringAliases(); assertEquals(queryBuilder, QueryBuilders.boolQuery() diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 53277bd5b3c0b..543931a497339 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -137,7 +137,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), + QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis) { @Override public MappedFieldType fieldMapper(String name) { diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 17c60152aaefc..831a7a41f6bdb 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -55,6 +55,7 @@ import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptServiceTests.TestEngineService; import org.elasticsearch.script.ScriptSettings; @@ -70,7 +71,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collections; -import java.util.Map; import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; @@ -93,9 +93,9 @@ public static void init() throws IOException { Environment environment = new Environment(baseSettings); ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new TestEngineService())); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - scriptService = new ScriptService(baseSettings, environment, - new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) { + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, null, scriptContextRegistry); + scriptService = new ScriptService(baseSettings, environment, new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, + scriptContextRegistry, scriptSettings, new ScriptMetrics()) { @Override public CompiledScript compile(Script script, ScriptContext scriptContext) { return new CompiledScript(ScriptType.INLINE, "mockName", "test", script); @@ -208,7 +208,7 @@ public void onCache(ShardId shardId, Accountable accountable) { } }); long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService, + return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService, null, xContentRegistry(), null, null, () -> nowInMillis) { @Override public MappedFieldType fieldMapper(String name) { @@ -251,7 +251,6 @@ protected static QueryBuilder randomNestedFilter() { } } - @SuppressWarnings("unchecked") private T copy(T original) throws IOException { /* The cast below is required to make Java 9 happy. Java 8 infers the T in copyWriterable to be the same as AbstractSortTestCase's * T but Java 9 infers it to be SortBuilder. */ diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java index 82f06441ef6da..d4e88283c59d2 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java @@ -22,7 +22,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.ReduceSearchPhaseException; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -34,8 +33,9 @@ import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.ScriptEngineService; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.script.TemplateService; +import org.elasticsearch.script.TemplateService.Backend; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder; import org.elasticsearch.search.suggest.phrase.Laplace; @@ -1112,12 +1112,12 @@ protected Collection> nodePlugins() { public static class DummyTemplatePlugin extends Plugin implements ScriptPlugin { @Override - public ScriptEngineService getScriptEngineService(Settings settings) { + public Backend getTemplateBackend() { return new DummyTemplateScriptEngine(); } } - public static class DummyTemplateScriptEngine implements ScriptEngineService { + public static class DummyTemplateScriptEngine implements TemplateService.Backend { // The collate query setter is hard coded to use mustache, so lets lie in this test about the script plugin, // which makes the collate code thinks mustache is evaluating the query. diff --git a/docs/reference/migration/migrate_6_0/scripting.asciidoc b/docs/reference/migration/migrate_6_0/scripting.asciidoc index 6740200922327..c7973aea786de 100644 --- a/docs/reference/migration/migrate_6_0/scripting.asciidoc +++ b/docs/reference/migration/migrate_6_0/scripting.asciidoc @@ -12,3 +12,18 @@ Use painless instead. milliseconds since epoch as a `long`. The same is true for `doc.some_date_field[some_number]`. Use `doc.some_date_field.value.millis` to fetch the milliseconds since epoch if you need it. + +==== Scripts and Templates are separate things now + +Internally we've split scripts and templates apart. This shouldn't matter over +the REST interface except for two things: +* During the rolling upgrade from 5.x to 6.0 the REST APIs for templates aren't +going to work properly most of the time. Don't use them. +* You can no longer PUT/GET/DELETE templates using the script API and you can +no longer PUT/GET/DELETE scripts using the template API. + +Transport client users will need to use some new actions in the lang-mustache +module to work with templates: +* `PutStoredTemplateAction` +* `GetStoredTemplateAction` +* `DeleteStoredTemplateAction` diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 9315a0fbd4b1f..90fd885b68148 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -33,7 +33,16 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.TemplateService.Backend; +import org.elasticsearch.script.mustache.stored.DeleteStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.GetStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.PutStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.RestDeleteStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.RestGetStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.RestPutStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.TransportDeleteStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.TransportGetStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.TransportPutStoredSearchTemplateAction; import java.util.Arrays; import java.util.List; @@ -42,16 +51,19 @@ import static java.util.Collections.singletonList; public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin, SearchPlugin { - @Override - public ScriptEngineService getScriptEngineService(Settings settings) { + public Backend getTemplateBackend() { return new MustacheScriptEngineService(); } @Override public List> getActions() { - return Arrays.asList(new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class), - new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class)); + return Arrays.asList( + new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class), + new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class), + new ActionHandler<>(DeleteStoredTemplateAction.INSTANCE, TransportDeleteStoredTemplateAction.class), + new ActionHandler<>(GetStoredTemplateAction.INSTANCE, TransportGetStoredTemplateAction.class), + new ActionHandler<>(PutStoredTemplateAction.INSTANCE, TransportPutStoredSearchTemplateAction.class)); } @Override @@ -66,9 +78,9 @@ public List getRestHandlers(Settings settings, RestController restC return Arrays.asList( new RestSearchTemplateAction(settings, restController), new RestMultiSearchTemplateAction(settings, restController), - new RestGetSearchTemplateAction(settings, restController), - new RestPutSearchTemplateAction(settings, restController), - new RestDeleteSearchTemplateAction(settings, restController), + new RestGetStoredTemplateAction(settings, restController), + new RestPutStoredTemplateAction(settings, restController), + new RestDeleteStoredTemplateAction(settings, restController), new RestRenderSearchTemplateAction(settings, restController)); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java index 2d39eb080e004..48b652d0fefc4 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java @@ -34,8 +34,8 @@ import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptEngineService; import org.elasticsearch.script.SearchScript; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.lookup.SearchLookup; import java.io.Reader; @@ -53,7 +53,7 @@ * process: First compile the string representing the template, the resulting * {@link Mustache} object can then be re-used for subsequent executions. */ -public final class MustacheScriptEngineService implements ScriptEngineService { +public final class MustacheScriptEngineService implements TemplateService.Backend { private static final Logger logger = ESLoggerFactory.getLogger(MustacheScriptEngineService.class); public static final String NAME = "mustache"; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index d7b0406238278..a332ca9315bb4 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -34,33 +34,27 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Collections; - +import static java.util.Collections.emptyMap; import static org.elasticsearch.script.ScriptContext.Standard.SEARCH; public class TransportSearchTemplateAction extends HandledTransportAction { - - private static final String TEMPLATE_LANG = MustacheScriptEngineService.NAME; - - private final ScriptService scriptService; + private final TemplateService templateService; private final TransportSearchAction searchAction; private final NamedXContentRegistry xContentRegistry; @Inject public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver resolver, - ScriptService scriptService, + TemplateService templateService, TransportSearchAction searchAction, NamedXContentRegistry xContentRegistry) { super(settings, SearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver, SearchTemplateRequest::new); - this.scriptService = scriptService; + this.templateService = templateService; this.searchAction = searchAction; this.xContentRegistry = xContentRegistry; } @@ -69,11 +63,10 @@ public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, T protected void doExecute(SearchTemplateRequest request, ActionListener listener) { final SearchTemplateResponse response = new SearchTemplateResponse(); try { - Script script = new Script(request.getScriptType(), TEMPLATE_LANG, request.getScript(), - request.getScriptParams() == null ? Collections.emptyMap() : request.getScriptParams()); - ExecutableScript executable = scriptService.executable(script, SEARCH); - - BytesReference source = (BytesReference) executable.run(); + BytesReference source = templateService + .template(request.getScript(), request.getScriptType(), SEARCH, null) + .apply(request.getScriptParams() == null ? + emptyMap() : request.getScriptParams()); response.setSource(source); if (request.isSimulate()) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateAction.java new file mode 100644 index 0000000000000..6b161086d0b46 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateAction.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class DeleteStoredTemplateAction extends Action< + DeleteStoredTemplateRequest, + DeleteStoredTemplateResponse, + DeleteStoredTemplateRequestBuilder> { + + public static final DeleteStoredTemplateAction INSTANCE = + new DeleteStoredTemplateAction(); + public static final String NAME = "cluster:admin/search/template/delete"; + + private DeleteStoredTemplateAction() { + super(NAME); + } + + @Override + public DeleteStoredTemplateResponse newResponse() { + return new DeleteStoredTemplateResponse(); + } + + @Override + public DeleteStoredTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new DeleteStoredTemplateRequestBuilder(client, this); + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequest.java new file mode 100644 index 0000000000000..449862086fa3e --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequest.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class DeleteStoredTemplateRequest extends AcknowledgedRequest< + DeleteStoredTemplateRequest> { + + private String id; + + DeleteStoredTemplateRequest() { + } + + public DeleteStoredTemplateRequest(String id) { + this.id = id; + } + + public DeleteStoredTemplateRequest id(String id) { + this.id = id; + return this; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (id == null || id.isEmpty()) { + validationException = addValidationError("must specify id for stored search template", + validationException); + } else + if (id.contains("#")) { + validationException = addValidationError( + "id cannot contain '#' for stored search template", validationException); + } + + return validationException; + } + + public String id() { + return id; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + + id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + + out.writeString(id); + } + + @Override + public String toString() { + return "delete stored script {id [" + id + "]}"; + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequestBuilder.java new file mode 100644 index 0000000000000..8103891df2fb3 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateRequestBuilder.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +public class DeleteStoredTemplateRequestBuilder extends AcknowledgedRequestBuilder< + DeleteStoredTemplateRequest, + DeleteStoredTemplateResponse, + DeleteStoredTemplateRequestBuilder> { + + public DeleteStoredTemplateRequestBuilder(ElasticsearchClient client, + DeleteStoredTemplateAction action) { + super(client, action, new DeleteStoredTemplateRequest()); + } + + public DeleteStoredTemplateRequestBuilder setId(String id) { + request.id(id); + return this; + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateResponse.java new file mode 100644 index 0000000000000..aab2172b2e735 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/DeleteStoredTemplateResponse.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class DeleteStoredTemplateResponse extends AcknowledgedResponse { + + DeleteStoredTemplateResponse() { + } + + public DeleteStoredTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateAction.java new file mode 100644 index 0000000000000..6ae26a4d97b03 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateAction.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class GetStoredTemplateAction extends Action< + GetStoredTemplateRequest, + GetStoredTemplateResponse, + GetStoredTemplateRequestBuilder> { + + public static final GetStoredTemplateAction INSTANCE = + new GetStoredTemplateAction(); + public static final String NAME = "cluster:admin/search/template/get"; + + private GetStoredTemplateAction() { + super(NAME); + } + + @Override + public GetStoredTemplateResponse newResponse() { + return new GetStoredTemplateResponse(); + } + + @Override + public GetStoredTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new GetStoredTemplateRequestBuilder(client, this); + } + +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequest.java new file mode 100644 index 0000000000000..1ae2a68d85a01 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequest.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class GetStoredTemplateRequest extends MasterNodeReadRequest< + GetStoredTemplateRequest> { + + protected String id; + + GetStoredTemplateRequest() { + } + + public GetStoredTemplateRequest(String id) { + this.id = id; + } + + public GetStoredTemplateRequest id(String id) { + this.id = id; + return this; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (id == null || id.isEmpty()) { + validationException = addValidationError("must specify id for stored search template", + validationException); + } else + if (id.contains("#")) { + validationException = addValidationError( + "id cannot contain '#' for stored search template", validationException); + } + + return validationException; + } + + public String id() { + return id; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(id); + } + + @Override + public String toString() { + return "get stored search template [" + id + "]"; + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequestBuilder.java new file mode 100644 index 0000000000000..61436840fa5c0 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateRequestBuilder.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +public class GetStoredTemplateRequestBuilder extends MasterNodeReadOperationRequestBuilder< + GetStoredTemplateRequest, + GetStoredTemplateResponse, + GetStoredTemplateRequestBuilder> { + + public GetStoredTemplateRequestBuilder(ElasticsearchClient client, + GetStoredTemplateAction action) { + super(client, action, new GetStoredTemplateRequest()); + } + + public GetStoredTemplateRequestBuilder setId(String id) { + request.id(id); + return this; + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateResponse.java new file mode 100644 index 0000000000000..9a60c60fd76c3 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/GetStoredTemplateResponse.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.script.StoredScriptSource; + +import java.io.IOException; + +public class GetStoredTemplateResponse extends ActionResponse implements ToXContent { + + private StoredScriptSource source; + + GetStoredTemplateResponse() { + } + + GetStoredTemplateResponse(StoredScriptSource source) { + this.source = source; + } + + /** + * @return if a stored script and if not found null + */ + public StoredScriptSource getSource() { + return source; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + source.toXContent(builder, params); + + return builder; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + + if (in.readBoolean()) { + source = new StoredScriptSource(in); + } else { + source = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + + if (source == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + source.writeTo(out); + } + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateAction.java new file mode 100644 index 0000000000000..bef0390ed681e --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateAction.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class PutStoredTemplateAction extends Action< + PutStoredTemplateRequest, + PutStoredTemplateResponse, + PutStoredTemplateRequestBuilder> { + + public static final PutStoredTemplateAction INSTANCE = + new PutStoredTemplateAction(); + public static final String NAME = "cluster:admin/search/template/put"; + + private PutStoredTemplateAction() { + super(NAME); + } + + @Override + public PutStoredTemplateResponse newResponse() { + return new PutStoredTemplateResponse(); + } + + @Override + public PutStoredTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new PutStoredTemplateRequestBuilder(client, this); + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequest.java new file mode 100644 index 0000000000000..f5d9f75e4376b --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequest.java @@ -0,0 +1,124 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class PutStoredTemplateRequest extends AcknowledgedRequest< + PutStoredTemplateRequest> { + private String id; + private BytesReference content; + private XContentType xContentType; + + public PutStoredTemplateRequest() { + } + + public PutStoredTemplateRequest(String id, BytesReference content, + XContentType xContentType) { + this.id = id; + this.content = content; + this.xContentType = Objects.requireNonNull(xContentType); + } + + public PutStoredTemplateRequest id(String id) { + this.id = id; + return this; + } + + public PutStoredTemplateRequest content(BytesReference content, + XContentType xContentType) { + this.content = content; + this.xContentType = xContentType; + return this; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (id == null || id.isEmpty()) { + validationException = addValidationError("must specify id for stored search template", + validationException); + } else + if (id.contains("#")) { + validationException = addValidationError( + "id cannot contain '#' for stored search template", validationException); + } + + if (content == null) { + validationException = addValidationError("must specify code for stored search template", + validationException); + } + + return validationException; + } + + public String id() { + return id; + } + + public BytesReference content() { + return content; + } + + public XContentType xContentType() { + return xContentType; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + id = in.readOptionalString(); + content = in.readBytesReference(); + xContentType = XContentType.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(id); + out.writeBytesReference(content); + xContentType.writeTo(out); + } + + @Override + public String toString() { + String source = "_na_"; + + try { + source = XContentHelper.convertToJson(content, false, xContentType); + } catch (Exception e) { + // ignore + } + + return "put search template {id [" + id + "], content [" + source + "]}"; + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequestBuilder.java new file mode 100644 index 0000000000000..91cbb2399d6b2 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateRequestBuilder.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentType; + +public class PutStoredTemplateRequestBuilder extends AcknowledgedRequestBuilder< + PutStoredTemplateRequest, + PutStoredTemplateResponse, + PutStoredTemplateRequestBuilder> { + + public PutStoredTemplateRequestBuilder(ElasticsearchClient client, + PutStoredTemplateAction action) { + super(client, action, new PutStoredTemplateRequest()); + } + + public PutStoredTemplateRequestBuilder setId(String id) { + request.id(id); + return this; + } + + public PutStoredTemplateRequestBuilder setContent(BytesReference content, + XContentType xContentType) { + request.content(content, xContentType); + return this; + } + +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateResponse.java new file mode 100644 index 0000000000000..4415227c7744d --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/PutStoredTemplateResponse.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +public class PutStoredTemplateResponse extends AcknowledgedResponse { + PutStoredTemplateResponse() { + } + + public PutStoredTemplateResponse(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestDeleteStoredTemplateAction.java similarity index 70% rename from modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestDeleteStoredTemplateAction.java index 61a394462f9c8..2621ee8a2e5cf 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestDeleteStoredTemplateAction.java @@ -16,34 +16,34 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.script.mustache; +package org.elasticsearch.script.mustache.stored; -import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.AcknowledgedRestListener; -import org.elasticsearch.script.Script; import java.io.IOException; import static org.elasticsearch.rest.RestRequest.Method.DELETE; -public class RestDeleteSearchTemplateAction extends BaseRestHandler { +public class RestDeleteStoredTemplateAction extends BaseRestHandler { - public RestDeleteSearchTemplateAction(Settings settings, RestController controller) { + public RestDeleteStoredTemplateAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(DELETE, "/_search/template/{id}", this); } @Override - public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) + throws IOException { String id = request.param("id"); - DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG); - return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel)); + DeleteStoredTemplateRequest deleteRequest = new DeleteStoredTemplateRequest(id); + return channel -> client.execute(DeleteStoredTemplateAction.INSTANCE, deleteRequest, + new AcknowledgedRestListener<>(channel)); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestGetStoredTemplateAction.java similarity index 54% rename from modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestGetStoredTemplateAction.java index 7d1ed4b57a421..f8442cf40272c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestGetStoredTemplateAction.java @@ -16,10 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.script.mustache; +package org.elasticsearch.script.mustache.stored; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.settings.Settings; @@ -38,44 +36,51 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestGetSearchTemplateAction extends BaseRestHandler { +public class RestGetStoredTemplateAction extends BaseRestHandler { public static final ParseField _ID_PARSE_FIELD = new ParseField("_id"); public static final ParseField FOUND_PARSE_FIELD = new ParseField("found"); - public RestGetSearchTemplateAction(Settings settings, RestController controller) { + public RestGetStoredTemplateAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(GET, "/_search/template/{id}", this); } @Override - public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException { + public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) + throws IOException { String id = request.param("id"); - GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG); + GetStoredTemplateRequest getRequest = new GetStoredTemplateRequest(id); - return channel -> client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(GetStoredScriptResponse response, XContentBuilder builder) throws Exception { - builder.startObject(); - builder.field(_ID_PARSE_FIELD.getPreferredName(), id); + return channel -> client.execute(GetStoredTemplateAction.INSTANCE, getRequest, + new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(GetStoredTemplateResponse response, + XContentBuilder builder) throws Exception { + builder.startObject(); + builder.field("_id", id); - builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), Script.DEFAULT_TEMPLATE_LANG); + builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), + Script.DEFAULT_TEMPLATE_LANG); - StoredScriptSource source = response.getSource(); - boolean found = source != null; - builder.field(FOUND_PARSE_FIELD.getPreferredName(), found); + StoredScriptSource source = response.getSource(); + boolean found = source != null; + builder.field("found", found); - if (found) { - builder.field(StoredScriptSource.TEMPLATE_PARSE_FIELD.getPreferredName(), source.getCode()); - } + if (found) { + builder.field( + StoredScriptSource.TEMPLATE_PARSE_FIELD.getPreferredName(), + source.getCode()); + } - builder.endObject(); + builder.endObject(); - return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, builder); - } - }); + return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, + builder); + } + }); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestPutStoredTemplateAction.java similarity index 74% rename from modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java rename to modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestPutStoredTemplateAction.java index 83925f0ec03b6..6fb7a03d548e2 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/RestPutStoredTemplateAction.java @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.script.mustache; +package org.elasticsearch.script.mustache.stored; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -26,16 +25,15 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.AcknowledgedRestListener; -import org.elasticsearch.script.Script; import java.io.IOException; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; -public class RestPutSearchTemplateAction extends BaseRestHandler { +public class RestPutStoredTemplateAction extends BaseRestHandler { - public RestPutSearchTemplateAction(Settings settings, RestController controller) { + public RestPutStoredTemplateAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(POST, "/_search/template/{id}", this); @@ -43,11 +41,14 @@ public RestPutSearchTemplateAction(Settings settings, RestController controller) } @Override - public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) + throws IOException { String id = request.param("id"); BytesReference content = request.content(); - PutStoredScriptRequest put = new PutStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG, content, request.getXContentType()); - return channel -> client.admin().cluster().putStoredScript(put, new AcknowledgedRestListener<>(channel)); + PutStoredTemplateRequest put = new PutStoredTemplateRequest(id, content, + request.getXContentType()); + return channel -> client.execute(PutStoredTemplateAction.INSTANCE, put, + new AcknowledgedRestListener<>(channel)); } } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportDeleteStoredTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportDeleteStoredTemplateAction.java new file mode 100644 index 0000000000000..d333a4c8205c2 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportDeleteStoredTemplateAction.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptMetaData; +import org.elasticsearch.script.TemplateService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportDeleteStoredTemplateAction extends TransportMasterNodeAction< + DeleteStoredTemplateRequest, DeleteStoredTemplateResponse> { + + private final TemplateService templateService; + + @Inject + public TransportDeleteStoredTemplateAction(Settings settings, + TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + TemplateService templateService) { + super(settings, DeleteStoredTemplateAction.NAME, transportService, clusterService, + threadPool, actionFilters, indexNameExpressionResolver, + DeleteStoredTemplateRequest::new); + this.templateService = templateService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected DeleteStoredTemplateResponse newResponse() { + return new DeleteStoredTemplateResponse(); + } + + @Override + protected void masterOperation(DeleteStoredTemplateRequest request, ClusterState state, + ActionListener listener) throws Exception { + clusterService.submitStateUpdateTask("delete-search-template-" + request.id(), + new AckedClusterStateUpdateTask(request, + listener) { + @Override + protected DeleteStoredTemplateResponse newResponse(boolean acknowledged) { + return new DeleteStoredTemplateResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); + smd = ScriptMetaData.deleteStoredScript(smd, request.id(), + templateService.getTemplateLanguage()); + MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()) + .putCustom(ScriptMetaData.TYPE, smd); + return ClusterState.builder(currentState).metaData(mdb).build(); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(DeleteStoredTemplateRequest request, + ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportGetStoredTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportGetStoredTemplateAction.java new file mode 100644 index 0000000000000..4a4589d496fd7 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportGetStoredTemplateAction.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptMetaData; +import org.elasticsearch.script.TemplateService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportGetStoredTemplateAction extends TransportMasterNodeReadAction< + GetStoredTemplateRequest, GetStoredTemplateResponse> { + + private final TemplateService templateService; + + @Inject + public TransportGetStoredTemplateAction(Settings settings, + TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + TemplateService templateService) { + super(settings, GetStoredTemplateAction.NAME, transportService, clusterService, + threadPool, actionFilters, indexNameExpressionResolver, + GetStoredTemplateRequest::new); + this.templateService = templateService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected GetStoredTemplateResponse newResponse() { + return new GetStoredTemplateResponse(); + } + + @Override + protected void masterOperation(GetStoredTemplateRequest request, ClusterState state, + ActionListener listener) throws Exception { + ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE); + + if (scriptMetadata != null) { + listener.onResponse(new GetStoredTemplateResponse(scriptMetadata + .getStoredScript(request.id(), templateService.getTemplateLanguage()))); + } else { + listener.onResponse(null); + } + } + + @Override + protected ClusterBlockException checkBlock(GetStoredTemplateRequest request, + ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportPutStoredSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportPutStoredSearchTemplateAction.java new file mode 100644 index 0000000000000..c9218e95001a0 --- /dev/null +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/stored/TransportPutStoredSearchTemplateAction.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache.stored; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptException; +import org.elasticsearch.script.ScriptMetaData; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.StoredScriptSource; +import org.elasticsearch.script.TemplateService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class TransportPutStoredSearchTemplateAction extends TransportMasterNodeAction< + PutStoredTemplateRequest, PutStoredTemplateResponse> { + + private final TemplateService templateService; + private final int maxScriptSizeInBytes; + + @Inject + public TransportPutStoredSearchTemplateAction(Settings settings, + TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + TemplateService templateService) { + super(settings, PutStoredTemplateAction.NAME, transportService, clusterService, + threadPool, actionFilters, indexNameExpressionResolver, + PutStoredTemplateRequest::new); + this.templateService = templateService; + maxScriptSizeInBytes = ScriptService.SCRIPT_MAX_SIZE_IN_BYTES.get(settings); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected PutStoredTemplateResponse newResponse() { + return new PutStoredTemplateResponse(); + } + + @Override + protected void masterOperation(PutStoredTemplateRequest request, ClusterState state, + ActionListener listener) throws Exception { + if (request.content().length() > maxScriptSizeInBytes) { + throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + + maxScriptSizeInBytes + "] with size [" + request.content().length() + + "] for script [" + request.id() + "]"); + } + + StoredScriptSource source = StoredScriptSource.parse(templateService.getTemplateLanguage(), + request.content(), request.xContentType()); + try { + templateService.checkCompileBeforeStore(source); + } catch (IllegalArgumentException | ScriptException e) { + throw new IllegalArgumentException("failed to parse/compile stored search template [" + + request.id() + "]" + + (source.getCode() == null ? "" : " using code [" + source.getCode() + "]"), + e); + } + + clusterService.submitStateUpdateTask("put-search-template-" + request.id(), + new AckedClusterStateUpdateTask(request, + listener) { + + @Override + protected PutStoredTemplateResponse newResponse(boolean acknowledged) { + return new PutStoredTemplateResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE); + smd = ScriptMetaData.putStoredScript(smd, request.id(), source); + MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()) + .putCustom(ScriptMetaData.TYPE, smd); + + return ClusterState.builder(currentState).metaData(mdb).build(); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(PutStoredTemplateRequest request, + ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/ContentTypeTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/ContentTypeTests.java new file mode 100644 index 0000000000000..fdd770f4bf076 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/ContentTypeTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptContextRegistry; +import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptSettings; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.TemplateService; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonMap; + +/** + * Validates that {@code contentType} flows properly from {@link TemplateService} into + * {@link MustacheScriptEngineService} with some basic examples. + */ +public class ContentTypeTests extends ESTestCase { + private TemplateService templateService; + + @Before + public void setupTemplateService() throws IOException { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) + .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE, 10000) + .build(); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(emptyList()); + TemplateService.Backend backend = new MustacheScriptEngineService(); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(emptyList()); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, backend, + scriptContextRegistry); + templateService = new TemplateService(settings, new Environment(settings), null, backend, + scriptContextRegistry, scriptSettings, new ScriptMetrics()); + } + + public void testNullContentType() { + testCase("test \"test\" test", null); + } + + public void testPlainText() { + testCase("test \"test\" test", CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE); + } + + public void testJson() { + testCase("test \\\"test\\\" test", CustomMustacheFactory.JSON_MIME_TYPE); + } + + public void testJsonWithCharset() { + testCase("test \\\"test\\\" test", CustomMustacheFactory.JSON_MIME_TYPE_WITH_CHARSET); + } + + public void testFormEncoded() { + testCase("test %22test%22 test", CustomMustacheFactory.X_WWW_FORM_URLENCODED_MIME_TYPE); + } + + private void testCase(String expected, String contentType) { + assertEquals(expected, templateService + .template("test {{param}} test", ScriptType.INLINE, + ScriptContext.Standard.SEARCH, contentType) + .apply(singletonMap("param", "\"test\"")).utf8ToString()); + + } + +} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java index 265edb7d53bd4..dec76688b8e58 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.script.mustache; import com.github.mustachejava.Mustache; + import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.Script; @@ -52,7 +52,7 @@ public void testCreateEncoder() { assertThat(e.getMessage(), equalTo("No encoder found for MIME type [test]")); assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE_WITH_CHARSET), - instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); + instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.JSON_MIME_TYPE), instanceOf(CustomMustacheFactory.JsonEscapeEncoder.class)); assertThat(CustomMustacheFactory.createEncoder(CustomMustacheFactory.PLAIN_TEXT_MIME_TYPE), diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index d123b2f307ede..ce84b822ddebb 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -28,6 +28,10 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.mustache.stored.DeleteStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.GetStoredTemplateAction; +import org.elasticsearch.script.mustache.stored.GetStoredTemplateResponse; +import org.elasticsearch.script.mustache.stored.PutStoredTemplateAction; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; @@ -150,8 +154,7 @@ public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws } public void testIndexedTemplateClient() throws Exception { - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("testTemplate") .setContent(new BytesArray("{" + "\"template\":{" + @@ -163,8 +166,7 @@ public void testIndexedTemplateClient() throws Exception { "}"), XContentType.JSON)); - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("testTemplate").setContent(new BytesArray("{" + "\"template\":{" + " \"query\":{" + @@ -174,8 +176,8 @@ public void testIndexedTemplateClient() throws Exception { "}" + "}"), XContentType.JSON)); - GetStoredScriptResponse getResponse = client().admin().cluster() - .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get(); + GetStoredTemplateResponse getResponse = GetStoredTemplateAction.INSTANCE.newRequestBuilder(client()) + .setId("testTemplate").get(); assertNotNull(getResponse.getSource()); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); @@ -196,11 +198,9 @@ public void testIndexedTemplateClient() throws Exception { .get(); assertHitCount(searchResponse.getResponse(), 4); - assertAcked(client().admin().cluster() - .prepareDeleteStoredScript(MustacheScriptEngineService.NAME, "testTemplate")); + assertAcked(DeleteStoredTemplateAction.INSTANCE.newRequestBuilder(client()).setId("testTemplate")); - getResponse = client().admin().cluster() - .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get(); + getResponse = GetStoredTemplateAction.INSTANCE.newRequestBuilder(client()).setId("testTemplate").get(); assertNull(getResponse.getSource()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()) @@ -211,8 +211,7 @@ public void testIndexedTemplateClient() throws Exception { } public void testIndexedTemplate() throws Exception { - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("1a") .setContent(new BytesArray("{" + "\"template\":{" + @@ -224,8 +223,7 @@ public void testIndexedTemplate() throws Exception { "}" ), XContentType.JSON) ); - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("2") .setContent(new BytesArray("{" + "\"template\":{" + @@ -236,8 +234,7 @@ public void testIndexedTemplate() throws Exception { "}" + "}"), XContentType.JSON) ); - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("3") .setContent(new BytesArray("{" + "\"template\":{" + @@ -288,7 +285,7 @@ public void testIndexedTemplate() throws Exception { .get(); assertHitCount(searchResponse.getResponse(), 1); assertWarnings("use of [/mustache/2] for looking up" + - " stored scripts/templates has been deprecated, use only [2] instead"); + " stored templates has been deprecated, use only [2] instead"); Map vars = new HashMap<>(); vars.put("fieldParam", "bar"); @@ -312,8 +309,7 @@ public void testIndexedTemplateOverwrite() throws Exception { int iterations = randomIntBetween(2, 11); for (int i = 1; i < iterations; i++) { - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("git01") .setContent(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + "\"type\": \"ooophrase_prefix\"}}}}}"), XContentType.JSON)); @@ -332,8 +328,7 @@ public void testIndexedTemplateOverwrite() throws Exception { assertThat(e.getMessage(), containsString("[match] query does not support type ooophrase_prefix")); assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"); - assertAcked(client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) .setId("git01") .setContent(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," + "\"type\": \"phrase_prefix\"}}}}"), XContentType.JSON)); @@ -349,12 +344,9 @@ public void testIndexedTemplateOverwrite() throws Exception { public void testIndexedTemplateWithArray() throws Exception { String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}"; - assertAcked( - client().admin().cluster().preparePutStoredScript() - .setLang(MustacheScriptEngineService.NAME) - .setId("4") - .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON) - ); + assertAcked(PutStoredTemplateAction.INSTANCE.newRequestBuilder(client()) + .setId("4") + .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON)); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON)); bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON)); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java index 3b70c5df626a5..04f3a341e27f0 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -33,7 +32,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.internal.SearchContext; @@ -42,13 +40,12 @@ import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.function.Function; +import static java.util.Collections.singleton; import static org.hamcrest.Matchers.containsString; public class TemplateQueryBuilderTests extends AbstractQueryTestCase { @@ -69,39 +66,7 @@ public void checkWarning() { @Override protected Collection> getPlugins() { - return Arrays.asList(MustachePlugin.class, CustomScriptPlugin.class); - } - - public static class CustomScriptPlugin extends MockScriptPlugin { - - @Override - @SuppressWarnings("unchecked") - protected Map, Object>> pluginScripts() { - Map, Object>> scripts = new HashMap<>(); - - scripts.put("{ \"match_all\" : {}}", - s -> new BytesArray("{ \"match_all\" : {}}")); - - scripts.put("{ \"match_all\" : {\"_name\" : \"foobar\"}}", - s -> new BytesArray("{ \"match_all\" : {\"_name\" : \"foobar\"}}")); - - scripts.put("{\n" + - " \"term\" : {\n" + - " \"foo\" : {\n" + - " \"value\" : \"bar\",\n" + - " \"boost\" : 2.0\n" + - " }\n" + - " }\n" + - "}", s -> new BytesArray("{\n" + - " \"term\" : {\n" + - " \"foo\" : {\n" + - " \"value\" : \"bar\",\n" + - " \"boost\" : 2.0\n" + - " }\n" + - " }\n" + - "}")); - return scripts; - } + return singleton(MustachePlugin.class); } @Before diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml index 644319d50ec3c..8f1c581c79c07 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml @@ -13,7 +13,7 @@ - match: { nodes.$master.modules.0.name: lang-mustache } --- -"Indexed template": +"Stored template": - do: put_template: diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml index f6fc458a08ec2..6d26b2ca2d01c 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml @@ -149,8 +149,9 @@ body: { "id" : "1", "params" : { "my_value" : "value1_foo", "my_size" : 1 } } - match: { hits.total: 1 } +--- +"Missing file template": - do: - catch: /unable.to.find.file.script.\[simple1\].using.lang.\[mustache\]/ + catch: /unable.to.find.file.template.\[id=missing, contentType=text/plain\]/ search_template: - body: { "file" : "simple1"} - + body: { "file" : "missing"} diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java index d809fd3fa885e..1b7b06b4d3820 100644 --- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java +++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java @@ -35,6 +35,7 @@ import org.elasticsearch.plugins.DiscoveryPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.watcher.ResourceWatcherService; @@ -72,6 +73,7 @@ public Collection createComponents( ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, + TemplateService templateService, NamedXContentRegistry xContentRegistry) { final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings); final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]"); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index 95810e089a275..53a131b5a1ec4 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -23,14 +23,14 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptMetrics; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.script.mustache.MustacheScriptEngineService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.util.Arrays; -import java.util.Collections; +import static java.util.Collections.emptyList; public abstract class AbstractScriptTestCase extends ESTestCase { @@ -42,13 +42,16 @@ public void init() throws Exception { .put("path.home", createTempDir()) .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false) .build(); - ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(new MustacheScriptEngineService())); - ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); - ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); - - ScriptService scriptService = new ScriptService(settings, new Environment(settings), null, - scriptEngineRegistry, scriptContextRegistry, scriptSettings); - templateService = new InternalTemplateService(scriptService); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(emptyList()); + MustacheScriptEngineService mustache = new MustacheScriptEngineService(); + ScriptSettings scriptSettings = new ScriptSettings(new ScriptEngineRegistry(emptyList()), + mustache, scriptContextRegistry); + + org.elasticsearch.script.TemplateService esTemplateService = + new org.elasticsearch.script.TemplateService(settings, new Environment(settings), + null, mustache, scriptContextRegistry, + scriptSettings, new ScriptMetrics()); + templateService = new InternalTemplateService(esTemplateService); } } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml index 0e54ff0b7ad59..848950a78f685 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/10_pipeline_with_mustache_templates.yaml @@ -349,4 +349,4 @@ } - match: { error.header.processor_type: "set" } - match: { error.type: "general_script_exception" } - - match: { error.reason: "Failed to compile inline script [{{#join}}{{/join}}] using lang [mustache]" } + - match: { error.reason: "Failed to compile [{{#join}}{{/join}}]" } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index ad414db491fb9..d726a3fb6e6e2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -90,6 +90,7 @@ import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateService; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.internal.SearchContext; import org.joda.time.DateTime; @@ -1000,6 +1001,7 @@ private static class ServiceHolder implements Closeable { private final MapperService mapperService; private final BitsetFilterCache bitsetFilterCache; private final ScriptService scriptService; + private final TemplateService templateService; private final Client client; private final long nowInMillis = randomNonNegativeLong(); @@ -1031,6 +1033,7 @@ private static class ServiceHolder implements Closeable { AnalysisModule analysisModule = new AnalysisModule(new Environment(nodeSettings), emptyList()); IndexAnalyzers indexAnalyzers = analysisModule.getAnalysisRegistry().build(idxSettings); scriptService = scriptModule.getScriptService(); + templateService = scriptModule.getTemplateService(); similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, @@ -1080,7 +1083,7 @@ public void close() throws IOException { QueryShardContext createShardContext() { return new QueryShardContext(0, idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, - scriptService, xContentRegistry, this.client, null, () -> nowInMillis); + scriptService, templateService, xContentRegistry, this.client, null, () -> nowInMillis); } ScriptModule createScriptModule(List scriptPlugins) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index fa659e06fb214..49197e0421bed 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1152,7 +1152,7 @@ public static ScriptModule newTestScriptModule() { .build(); Environment environment = new Environment(settings); MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); - return new ScriptModule(settings, environment, null, singletonList(scriptEngine), emptyList()); + return new ScriptModule(settings, environment, null, singletonList(scriptEngine), emptyList(), null); } /** Creates an IndicesModule for testing with the given mappers and metadata mappers. */ diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index 5f7e38b7ec1ce..d01244c2399b4 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -40,7 +40,7 @@ public class MockSearchServiceTests extends ESTestCase { public void testAssertNoInFlightContext() { final long nowInMillis = randomNonNegativeLong(); SearchContext s = new TestSearchContext(new QueryShardContext(0, - new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null, null, null, null, null, xContentRegistry(), + new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis)) { @Override