Add support for Remove Duplicates token filter (#3426)

Author: Mpdreamz

src/Nest/Analysis/TokenFilters/RemoveDuplicatesTokenFilter.cs

@@ -0,0 +1,25 @@
+using System.Collections;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// A token filter that drops identical tokens at the same position
+	/// </summary>
+	public interface IRemoveDuplicatesTokenFilter : ITokenFilter { }
+
+	///<inheritdoc cref="IRemoveDuplicatesTokenFilter"/>
+	public class RemoveDuplicatesTokenFilter : TokenFilterBase, IRemoveDuplicatesTokenFilter
+	{
+		internal const string TokenType = "remove_duplicates";
+		public RemoveDuplicatesTokenFilter() : base(TokenType) { }
+	}
+
+	///<inheritdoc cref="IRemoveDuplicatesTokenFilter"/>
+	public class RemoveDuplicatesTokenFilterDescriptor
+		: TokenFilterDescriptorBase<RemoveDuplicatesTokenFilterDescriptor, IRemoveDuplicatesTokenFilter>, IRemoveDuplicatesTokenFilter
+	{
+		protected override string Type => RemoveDuplicatesTokenFilter.TokenType;
+	}
+}

src/Nest/Analysis/TokenFilters/TokenFilters.cs

@@ -324,5 +324,9 @@ public TokenFiltersDescriptor IcuTransform(string name, Func<IcuTransformTokenFi
 		/// </summary>
 		public TokenFiltersDescriptor Multiplexer(string name, Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
 			Assign(name, selector.Invoke(new MultiplexerTokenFilterDescriptor()));
+
+		/// <summary> A token filter of type remove_duplicates that drops identical tokens at the same position. </summary>
+		public TokenFiltersDescriptor RemoveDuplicates(string name, Func<RemoveDuplicatesTokenFilterDescriptor, IRemoveDuplicatesTokenFilter> selector = null) =>
+			Assign(name, selector.InvokeOrDefault(new RemoveDuplicatesTokenFilterDescriptor()));
 	}
 }

src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs

@@ -327,6 +327,10 @@ public AnalyzeTokenFiltersDescriptor IcuTransform(Func<IcuTransformTokenFilterDe
 		///<inheritdoc cref="IMultiplexerTokenFilter"/>
 		public AnalyzeTokenFiltersDescriptor Multiplexer(Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
 			AssignIfNotNull(selector.Invoke(new MultiplexerTokenFilterDescriptor()));
+
+		///<inheritdoc cref="IRemoveDuplicatesTokenFilter"/>
+		public AnalyzeTokenFiltersDescriptor RemoveDuplicates(Func<RemoveDuplicatesTokenFilterDescriptor, IRemoveDuplicatesTokenFilter> selector) =>
+			AssignIfNotNull(selector.Invoke(new RemoveDuplicatesTokenFilterDescriptor()));
 	}
 
 }

src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs

@@ -902,5 +902,14 @@ public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>
 					.PreserveOriginal()
 				);
 		}
+
+		[SkipVersion("<6.4.0", "Introduced in 6.4.0")]
+		public class RemoveDuplicatesTests : TokenFilterAssertionBase<PhoneticTests>
+		{
+			public override string Name => "dupes";
+			public override object Json => new { };
+			public override ITokenFilter Initializer => new RemoveDuplicatesTokenFilter { };
+			public override FuncTokenFilters Fluent => (n, tf) => tf.RemoveDuplicates(n);
+		}
 	}
 }