Expose dissect and grok to painless

libs/dissect/src/main/java/org/elasticsearch/dissect/DissectKey.java

@@ -102,7 +102,7 @@ public final class DissectKey {
         }
 
         if (name == null || (name.isEmpty() && !skip)) {
-            throw new DissectException.KeyParse(key, "The key name could be determined");
+            throw new DissectException.KeyParse(key, "The key name could not be determined");
         }
     }
 

libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java

@@ -34,7 +34,9 @@
 import java.util.stream.Collectors;
 
 /**
- * <p>Splits (dissects) a string into its parts based on a pattern.</p><p>A dissect pattern is composed of a set of keys and delimiters.
+ * Splits (dissects) a string into its parts based on a pattern.
+ * <p>
+ * A dissect pattern is composed of a set of keys and delimiters.
  * For example the dissect pattern: <pre>%{a} %{b},%{c}</pre> has 3 keys (a,b,c) and two delimiters (space and comma). This pattern will
  * match a string of the form: <pre>foo bar,baz</pre> and will result a key/value pairing of <pre>a=foo, b=bar, and c=baz.</pre>
  * <p>Matches are all or nothing. For example, the same pattern will NOT match <pre>foo bar baz</pre> since all of the delimiters did not
@@ -171,11 +173,9 @@ public DissectParser(String pattern, String appendSeparator) {
      *
      * @param inputString The string to dissect
      * @return the key/value Map of the results
-     * @throws DissectException if unable to dissect a pair into it's parts.
      */
     public Map<String, String> parse(String inputString) {
-        /**
-         *
+        /*
          * This implements a naive string matching algorithm. The string is walked left to right, comparing each byte against
          * another string's bytes looking for matches. If the bytes match, then a second cursor looks ahead to see if all the bytes
          * of the other string matches. If they all match, record it and advances the primary cursor to the match point. If it can not match
@@ -276,7 +276,19 @@ public Map<String, String> parse(String inputString) {
         }
         Map<String, String> results = dissectMatch.getResults();
 
-        if (!dissectMatch.isValid(results)) {
+        return dissectMatch.isValid(results) ? results : null;
+    }
+
+    /**
+     * <p>Entry point to dissect a string into it's parts.</p>
+     *
+     * @param inputString The string to dissect
+     * @return the key/value Map of the results
+     * @throws DissectException if unable to dissect a pair into it's parts.
+     */
+    public Map<String, String> forceParse(String inputString) {
+        Map<String, String> results = parse(inputString);
+        if (results == null) {
             throw new DissectException.FindMatch(pattern, inputString);
         }
         return results;

libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java

@@ -344,11 +344,12 @@ public void testJsonSpecification() throws Exception {
         }
     }
 
-    private DissectException assertFail(String pattern, String input){
-        return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).parse(input));
+    private DissectException assertFail(String pattern, String input) {
+        return expectThrows(DissectException.class, () -> new DissectParser(pattern, null).forceParse(input));
     }
 
     private void assertMiss(String pattern, String input) {
+        assertNull(new DissectParser(pattern, null).parse(input));
         DissectException e = assertFail(pattern, input);
         assertThat(e.getMessage(), CoreMatchers.containsString("Unable to find match for dissect pattern"));
         assertThat(e.getMessage(), CoreMatchers.containsString(pattern));

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java

@@ -54,7 +54,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
         } else if (input == null) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot process it.");
         }
-        dissectParser.parse(input).forEach(ingestDocument::setFieldValue);
+        dissectParser.forceParse(input).forEach(ingestDocument::setFieldValue);
         return ingestDocument;
     }
 

modules/lang-painless/build.gradle

@@ -40,6 +40,8 @@ dependencies {
   api 'org.ow2.asm:asm-commons:7.2'
   api 'org.ow2.asm:asm-analysis:7.2'
   api 'org.ow2.asm:asm:7.2'
+  api project(':libs:elasticsearch-grok')
+  api project(':libs:elasticsearch-dissect')
   api project('spi')
 }
 

modules/lang-painless/src/main/antlr/PainlessLexer.g4

@@ -109,7 +109,7 @@ INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fFdD]?;
 
 STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\'] )*? '\'' );
-REGEX: '/' ( '\\' ~'\n' | ~('/' | '\n') )+? '/' [cilmsUux]* { isSlashRegex() }?;
+REGEX: [dg]? '/' ( '\\' ~'\n' | ~('/' | '\n') )+? '/' [cilmsUux]* { isSlashRegex() }?;
 
 TRUE:  'true';
 FALSE: 'false';

modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.grok.MatcherWatchdog;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.ir.ClassNode;
 import org.elasticsearch.painless.lookup.PainlessLookup;
@@ -46,6 +47,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 
@@ -165,16 +167,29 @@ public Loader createLoader(ClassLoader parent) {
      */
     private final Map<String, Class<?>> additionalClasses;
 
+    /**
+     * Suppliers the watchdog that prevents grok from running forever.
+     */
+    private final Supplier<MatcherWatchdog> grokWatchdog;
+
     /**
      * Standard constructor.
      * @param scriptClass The class/interface the script will implement.
      * @param factoryClass An optional class/interface to create the {@code scriptClass} instance.
      * @param statefulFactoryClass An optional class/interface to create the {@code factoryClass} instance.
      * @param painlessLookup The whitelist the script will use.
+     * @param grokWatchdog Supplies the watchdog used to prevent grok from running forever
      */
-    Compiler(Class<?> scriptClass, Class<?> factoryClass, Class<?> statefulFactoryClass, PainlessLookup painlessLookup) {
+    Compiler(
+        Class<?> scriptClass,
+        Class<?> factoryClass,
+        Class<?> statefulFactoryClass,
+        PainlessLookup painlessLookup,
+        Supplier<MatcherWatchdog> grokWatchdog
+    ) {
         this.scriptClass = scriptClass;
         this.painlessLookup = painlessLookup;
+        this.grokWatchdog = grokWatchdog;
         Map<String, Class<?>> additionalClasses = new HashMap<>();
         additionalClasses.put(scriptClass.getName(), scriptClass);
         addFactoryMethod(additionalClasses, factoryClass, "newInstance");
@@ -218,7 +233,15 @@ ScriptScope compile(Loader loader, String name, String source, CompilerSettings
         String scriptName = Location.computeSourceName(name);
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptName, source, settings);
-        ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1);
+        ScriptScope scriptScope = new ScriptScope(
+            painlessLookup,
+            settings,
+            scriptClassInfo,
+            scriptName,
+            source,
+            grokWatchdog,
+            root.getIdentifier() + 1
+        );
         new PainlessSemanticHeaderPhase().visitClass(root, scriptScope);
         new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope);
         // TODO: Make this phase optional #60156
@@ -254,7 +277,15 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de
         String scriptName = Location.computeSourceName(name);
         ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass);
         SClass root = Walker.buildPainlessTree(scriptName, source, settings);
-        ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1);
+        ScriptScope scriptScope = new ScriptScope(
+            painlessLookup,
+            settings,
+            scriptClassInfo,
+            scriptName,
+            source,
+            grokWatchdog,
+            root.getIdentifier() + 1
+        );
         new PainlessSemanticHeaderPhase().visitClass(root, scriptScope);
         new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope);
         // TODO: Make this phase optional #60156

modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.grok.Grok;
 import org.elasticsearch.painless.api.Augmentation;
 
 import java.util.HashMap;
@@ -77,6 +78,9 @@ public final class CompilerSettings {
      * For testing. Do not use.
      */
     private int initialCallSiteDepth = 0;
+
+    private Map<String, String> grokPatternBank = Grok.BUILTIN_PATTERNS;
+
     private int testInject0 = 2;
     private int testInject1 = 4;
     private int testInject2 = 6;
@@ -170,6 +174,20 @@ public int getRegexLimitFactor() {
         return regexLimitFactor;
     }
 
+    /**
+     * Default grok "pattern bank". Mostly initialized here so 
+     */
+    public Map<String, String> getGrokPatternBank() {
+        return grokPatternBank;
+    }
+
+    public void addToGrokPatternBank(String name, String pattern) {
+        if (grokPatternBank == Grok.BUILTIN_PATTERNS) {
+            grokPatternBank = new HashMap<>(grokPatternBank);
+        }
+        grokPatternBank.put(name, pattern);
+    }
+
     /**
      * Get compiler settings as a map.  This is used to inject compiler settings into augmented methods with the {@code @inject_constant}
      * annotation.

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java

@@ -33,9 +33,12 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.LazyInitializable;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.grok.MatcherWatchdog;
 import org.elasticsearch.painless.action.PainlessContextAction;
 import org.elasticsearch.painless.action.PainlessExecuteAction;
 import org.elasticsearch.painless.spi.PainlessExtension;
@@ -111,6 +114,8 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
     }
 
     private final SetOnce<PainlessScriptEngine> painlessScriptEngine = new SetOnce<>();
+    private final SetOnce<ThreadPool> threadPool = new SetOnce<>();
+    private final Supplier<MatcherWatchdog> grokWatchdog = new LazyInitializable<>(this::initGrokWatchdog)::getOrCompute;
 
     @Override
     public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
@@ -123,7 +128,7 @@ public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<
             }
             contextsWithWhitelists.put(context, contextWhitelists);
         }
-        painlessScriptEngine.set(new PainlessScriptEngine(settings, contextsWithWhitelists));
+        painlessScriptEngine.set(new PainlessScriptEngine(settings, contextsWithWhitelists, grokWatchdog));
         return painlessScriptEngine.get();
     }
 
@@ -136,6 +141,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
                                                Supplier<RepositoriesService> repositoriesServiceSupplier) {
         // this is a hack to bind the painless script engine in guice (all components are added to guice), so that
         // the painless context api. this is a temporary measure until transport actions do no require guice
+        this.threadPool.set(threadPool);
         return Collections.singletonList(painlessScriptEngine.get());
     }
 
@@ -178,4 +184,13 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         handlers.add(new PainlessContextAction.RestAction());
         return handlers;
     }
+
+    private MatcherWatchdog initGrokWatchdog() {
+        // TODO this is fairly unpleasant
+        ThreadPool threadPool = this.threadPool.get();
+        return MatcherWatchdog.newInstance(1000, 1000, threadPool::relativeTimeInMillis,
+            (delay, command) -> threadPool.schedule(
+                command, TimeValue.timeValueMillis(delay), ThreadPool.Names.GENERIC
+            ));
+    }
 }

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.grok.MatcherWatchdog;
 import org.elasticsearch.painless.Compiler.Loader;
 import org.elasticsearch.painless.lookup.PainlessLookup;
 import org.elasticsearch.painless.lookup.PainlessLookupBuilder;
@@ -45,9 +46,11 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.painless.WriterConstants.OBJECT_TYPE;
 
@@ -79,7 +82,7 @@ public final class PainlessScriptEngine implements ScriptEngine {
 
     /**
      * Default compiler settings to be used. Note that {@link CompilerSettings} is mutable but this instance shouldn't be mutated outside
-     * of {@link PainlessScriptEngine#PainlessScriptEngine(Settings, Map)}.
+     * of {@link PainlessScriptEngine#PainlessScriptEngine}.
      */
     private final CompilerSettings defaultCompilerSettings = new CompilerSettings();
 
@@ -90,7 +93,11 @@ public final class PainlessScriptEngine implements ScriptEngine {
      * Constructor.
      * @param settings The settings to initialize the engine with.
      */
-    public PainlessScriptEngine(Settings settings, Map<ScriptContext<?>, List<Whitelist>> contexts) {
+    public PainlessScriptEngine(
+        Settings settings,
+        Map<ScriptContext<?>, List<Whitelist>> contexts,
+        Supplier<MatcherWatchdog> grokWatchdog
+    ) {
         defaultCompilerSettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(settings));
         defaultCompilerSettings.setRegexLimitFactor(CompilerSettings.REGEX_LIMIT_FACTOR.get(settings));
 
@@ -101,7 +108,7 @@ public PainlessScriptEngine(Settings settings, Map<ScriptContext<?>, List<Whitel
             ScriptContext<?> context = entry.getKey();
             PainlessLookup lookup = PainlessLookupBuilder.buildFromWhitelists(entry.getValue());
             contextsToCompilers.put(context,
-                    new Compiler(context.instanceClazz, context.factoryClazz, context.statefulFactoryClazz, lookup));
+                    new Compiler(context.instanceClazz, context.factoryClazz, context.statefulFactoryClazz, lookup, grokWatchdog));
             contextsToLookups.put(context, lookup);
         }
 
@@ -449,6 +456,15 @@ private CompilerSettings buildCompilerSettings(Map<String, String> params) {
                 compilerSettings.setInitialCallSiteDepth(Integer.parseInt(value));
             }
 
+            for (Iterator<Map.Entry<String, String>> itr = copy.entrySet().iterator(); itr.hasNext();) {
+                Map.Entry<String, String> e = itr.next();
+                if (false == e.getKey().startsWith("grok.pattern.")) {
+                    continue;
+                }
+                itr.remove();
+                compilerSettings.addToGrokPatternBank(e.getKey().substring("grok.pattern.".length()), e.getValue());
+            }
+
             value = copy.remove(CompilerSettings.REGEX_ENABLED.getKey());
             if (value != null) {
                 throw new IllegalArgumentException("[painless.regex.enabled] can only be set on node startup.");