Reset Token position on reuse in scripted analysis (#47424)

Most of the information in AnalysisPredicateScript.Token is pulled directly
from its underlying AttributeSource, but we also keep track of the token position,
and this state is held directly on the Token. This information needs to be reset when
the containing ScriptFilteringTokenFilter or ScriptedConditionTokenFilter is re-used.

Fixes #47197

Author: romseygeek

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java

@@ -61,6 +61,10 @@ public Token(AttributeSource source) {
             this.keywordAtt = source.addAttribute(KeywordAttribute.class);
         }
 
+        public void reset() {
+            this.pos = -1;
+        }
+
         public void updatePosition() {
             this.pos = this.pos + posIncAtt.getPositionIncrement();
         }

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/PredicateTokenFilterScriptFactory.java

@@ -69,5 +69,11 @@ protected boolean accept() throws IOException {
             token.updatePosition();
             return script.execute(token);
         }
+
+        @Override
+        public void reset() throws IOException {
+            super.reset();
+            this.token.reset();
+        }
     }
 }

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptType;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Function;
@@ -119,6 +120,12 @@ protected boolean shouldFilter() {
             token.updatePosition();
             return script.execute(token);
         }
+
+        @Override
+        public void reset() throws IOException {
+            super.reset();
+            token.reset();
+        }
     }
 
 }

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java

@@ -46,7 +46,7 @@ public void testSimpleFilter() throws IOException {
         Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put("index.analysis.filter.f.type", "predicate_token_filter")
-            .put("index.analysis.filter.f.script.source", "token.getTerm().length() > 5")
+            .put("index.analysis.filter.f.script.source", "my_script")
             .put("index.analysis.analyzer.myAnalyzer.type", "custom")
             .put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard")
             .putList("index.analysis.analyzer.myAnalyzer.filter", "f")
@@ -56,7 +56,7 @@ public void testSimpleFilter() throws IOException {
         AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() {
             @Override
             public boolean execute(Token token) {
-                return token.getTerm().length() > 5;
+                return token.getPosition() < 2 || token.getPosition() > 4;
             }
         };
 
@@ -65,7 +65,7 @@ public boolean execute(Token token) {
             @Override
             public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
                 assertEquals(context, AnalysisPredicateScript.CONTEXT);
-                assertEquals(new Script("token.getTerm().length() > 5"), script);
+                assertEquals(new Script("my_script"), script);
                 return (FactoryType) factory;
             }
         };
@@ -79,8 +79,8 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
 
         try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) {
             assertNotNull(analyzer);
-            assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[]{
-                "Vorsprung", "Technik"
+            assertAnalyzesTo(analyzer, "Oh what a wonderful thing to be", new String[]{
+                "Oh", "what", "to", "be"
             });
         }
 

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java

@@ -45,7 +45,7 @@ public void testSimpleCondition() throws Exception {
         Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put("index.analysis.filter.cond.type", "condition")
-            .put("index.analysis.filter.cond.script.source", "token.getTerm().length() > 5")
+            .put("index.analysis.filter.cond.script.source", "token.getPosition() > 1")
             .putList("index.analysis.filter.cond.filter", "uppercase")
             .put("index.analysis.analyzer.myAnalyzer.type", "custom")
             .put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard")
@@ -56,7 +56,7 @@ public void testSimpleCondition() throws Exception {
         AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() {
             @Override
             public boolean execute(Token token) {
-                return token.getTerm().length() > 5;
+                return token.getPosition() > 1;
             }
         };
 
@@ -65,7 +65,7 @@ public boolean execute(Token token) {
             @Override
             public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
                 assertEquals(context, AnalysisPredicateScript.CONTEXT);
-                assertEquals(new Script("token.getTerm().length() > 5"), script);
+                assertEquals(new Script("token.getPosition() > 1"), script);
                 return (FactoryType) factory;
             }
         };
@@ -80,7 +80,7 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) {
             assertNotNull(analyzer);
             assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[]{
-                "VORSPRUNG", "Durch", "TECHNIK"
+                "Vorsprung", "Durch", "TECHNIK"
             });
         }