From 5c3d04192f5b4b5b09f25e1fa8f00951c50453f1 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Tue, 18 Dec 2018 16:26:45 +0100 Subject: [PATCH 1/3] Add support for condition token filter elastic/elasticseach#31958 --- .../TokenFilters/ConditionTokenFilter.cs | 66 +++++++++++++++++++ .../Analysis/TokenFilters/TokenFilters.cs | 4 ++ .../Analysis/TokenFilters/TokenFilterTests.cs | 26 ++++++++ 3 files changed, 96 insertions(+) create mode 100644 src/Nest/Analysis/TokenFilters/ConditionTokenFilter.cs diff --git a/src/Nest/Analysis/TokenFilters/ConditionTokenFilter.cs b/src/Nest/Analysis/TokenFilters/ConditionTokenFilter.cs new file mode 100644 index 00000000000..8279d4d591e --- /dev/null +++ b/src/Nest/Analysis/TokenFilters/ConditionTokenFilter.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Nest +{ + /// + /// The conditional token filter takes a predicate script and a list of subfilters, and + /// only applies the subfilters to the current token if it matches the predicate. + /// + public interface IConditionTokenFilter : ITokenFilter + { + /// + /// a predicate script that determines whether or not the filters will be applied + /// to the current token. Note that only inline scripts are supported + /// + [JsonProperty("script")] + IScript Script { get; set; } + + /// + ///a chain of token filters to apply to the current token if the predicate + /// matches. These can be any token filters defined elsewhere in the index mappings. + /// + [JsonProperty("filter")] + IEnumerable Filters { get; set; } + } + + /// + public class ConditionTokenFilter : TokenFilterBase, IConditionTokenFilter + { + public ConditionTokenFilter() : base("condition") { } + + /// + public IScript Script { get; set; } + + /// + public IEnumerable Filters { get; set; } + } + + /// + public class ConditionTokenFilterDescriptor + : TokenFilterDescriptorBase, IConditionTokenFilter + { + protected override string Type => "condition"; + + IScript IConditionTokenFilter.Script { get; set; } + IEnumerable IConditionTokenFilter.Filters { get; set; } + + /// + public ConditionTokenFilterDescriptor Script(Func scriptSelector) => + Assign(a => a.Script = scriptSelector?.Invoke(new ScriptDescriptor())); + + /// + public ConditionTokenFilterDescriptor Script(string predicate) => + Assign(a => a.Script = new InlineScript(predicate)); + + /// + public ConditionTokenFilterDescriptor Filters(params string[] filters) => + Assign(a => a.Filters = filters); + + /// + public ConditionTokenFilterDescriptor Filters(IEnumerable filters) => + Assign(a => a.Filters = filters); + +} +} diff --git a/src/Nest/Analysis/TokenFilters/TokenFilters.cs b/src/Nest/Analysis/TokenFilters/TokenFilters.cs index 4fb95d3ce17..c223e7dafd9 100644 --- a/src/Nest/Analysis/TokenFilters/TokenFilters.cs +++ b/src/Nest/Analysis/TokenFilters/TokenFilters.cs @@ -225,6 +225,10 @@ public TokenFiltersDescriptor Standard(string name, Func selector) => Assign(name, selector?.Invoke(new StemmerTokenFilterDescriptor())); + /// + public TokenFiltersDescriptor Condition(string name, Func selector) => + Assign(name, selector?.Invoke(new ConditionTokenFilterDescriptor())); + /// /// Overrides stemming algorithms, by applying a custom mapping, then protecting these terms from being modified by stemmers. Must be placed /// before any stemming filters. diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs index 6f649e6509f..1aee75cbb20 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -896,6 +896,32 @@ public class NoriPartOfSpeechTests : TokenFilterAssertionBase "nori_pos"; } + [SkipVersion("<6.5.0", "Introduced in 6.5.0")] + public class ConditionTests : TokenFilterAssertionBase + { + private readonly string _predicate = "token.getTerm().length() < 5"; + + public override FuncTokenFilters Fluent => (n, tf) => tf + .Condition(n, t => t + .Filters("lowercase", "lowercase, porter_stem") + .Script(_predicate) + ); + + public override ITokenFilter Initializer => new ConditionTokenFilter + { + Filters = new[] { "lowercase", "lowercase, porter_stem" }, + Script = new InlineScript(_predicate) + }; + + public override object Json => new + { + filters = new[] { "lowercase", "lowercase, porter_stem" }, + script = new { source = _predicate } + }; + + public override string Name => "multiplexer"; + } + [SkipVersion("<6.4.0", "Introduced in 6.4.0")] public class MultiplexerTests : TokenFilterAssertionBase { From 19cb07986ee3df16cc17c2ac775ff80a0ae3bf92 Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Thu, 24 Jan 2019 14:51:35 +0100 Subject: [PATCH 2/3] fix failing unit tests --- src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs | 4 ++++ .../Tests/Analysis/TokenFilters/TokenFilterTests.cs | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs index ea43995bc8f..b6699b71481 100644 --- a/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs +++ b/src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs @@ -241,6 +241,10 @@ public AnalyzeTokenFiltersDescriptor Standard(Func + public AnalyzeTokenFiltersDescriptor Condition(Func selector) => + AssignIfNotNull(selector?.Invoke(new ConditionTokenFilterDescriptor())); + /// /// A filter that stems words (similar to snowball, but with more options). /// diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs index 1aee75cbb20..fbea73d448d 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -915,15 +915,16 @@ public class ConditionTests : TokenFilterAssertionBase public override object Json => new { - filters = new[] { "lowercase", "lowercase, porter_stem" }, + type = "condition", + filter = new[] { "lowercase", "lowercase, porter_stem" }, script = new { source = _predicate } }; - public override string Name => "multiplexer"; + public override string Name => "condition"; } [SkipVersion("<6.4.0", "Introduced in 6.4.0")] - public class MultiplexerTests : TokenFilterAssertionBase + public class MultiplexerTests : TokenFilterAssertionBase { public override FuncTokenFilters Fluent => (n, tf) => tf .Multiplexer(n, t => t @@ -939,6 +940,7 @@ public class MultiplexerTests : TokenFilterAssertionBase public override object Json => new { + type = "multiplexer", filters = new[] { "lowercase", "lowercase, porter_stem" }, preserve_original = true }; @@ -947,7 +949,7 @@ public class MultiplexerTests : TokenFilterAssertionBase } [SkipVersion("<6.4.0", "Introduced in 6.4.0")] - public class RemoveDuplicatesTests : TokenFilterAssertionBase + public class RemoveDuplicatesTests : TokenFilterAssertionBase { public override FuncTokenFilters Fluent => (n, tf) => tf.RemoveDuplicates(n); public override ITokenFilter Initializer => new RemoveDuplicatesTokenFilter { }; From bae20e80659647ae4afc59cb3328e04abd48a57c Mon Sep 17 00:00:00 2001 From: Mpdreamz Date: Thu, 24 Jan 2019 15:05:43 +0100 Subject: [PATCH 3/3] fix failing unit tests --- src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs index fbea73d448d..b64c9c60de0 100644 --- a/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs +++ b/src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs @@ -953,7 +953,7 @@ public class RemoveDuplicatesTests : TokenFilterAssertionBase (n, tf) => tf.RemoveDuplicates(n); public override ITokenFilter Initializer => new RemoveDuplicatesTokenFilter { }; - public override object Json => new { }; + public override object Json => new { type = "remove_duplicates" }; public override string Name => "dupes"; } }