Add ignore_missing to ingest processors (#3623)

This commit adds missing fields for ingest processors,
including ignore_missing and target_field

Fixes #3619

Author: russcam

src/Nest/Analysis/Plugins/Icu/IcuAnalyzer.cs

@@ -46,9 +46,9 @@ public class IcuAnalyzerDescriptor : AnalyzerDescriptorBase<IcuAnalyzerDescripto
 		IcuNormalizationMode? IIcuAnalyzer.Mode { get; set; }
 
 		/// <inheritdoc cref="IIcuAnalyzer.Method"/>
-		public IcuAnalyzerDescriptor Method(IcuNormalizationType? method) => Assign(a => a.Method = method);
+		public IcuAnalyzerDescriptor Method(IcuNormalizationType? method) => Assign(method, (a, v) => a.Method = v);
 
 		/// <inheritdoc cref="IIcuAnalyzer.Mode"/>
-		public IcuAnalyzerDescriptor Mode(IcuNormalizationMode? mode) => Assign(a => a.Mode = mode);
+		public IcuAnalyzerDescriptor Mode(IcuNormalizationMode? mode) => Assign(mode, (a, v) => a.Mode = v);
 	}
 }

src/Nest/Ingest/Processors/ConvertProcessor.cs

@@ -6,25 +6,52 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// Converts a field in the currently ingested document to a different type,
+	/// such as converting a string to an integer.
+	/// If the field value is an array, all members will be converted.
+	/// </summary>
 	[JsonObject(MemberSerialization.OptIn)]
 	[JsonConverter(typeof(ProcessorJsonConverter<ConvertProcessor>))]
 	public interface IConvertProcessor : IProcessor
 	{
+		/// <summary>
+		/// The field whose value is to be converted
+		/// </summary>
 		[JsonProperty("field")]
 		Field Field { get; set; }
 
+		/// <summary>
+		/// The field to assign the converted value to, by default field is updated in-place
+		/// </summary>
 		[JsonProperty("target_field")]
 		Field TargetField { get; set; }
 
+		/// <summary>
+		/// The type to convert the existing value to
+		/// </summary>
 		[JsonProperty("type")]
 		ConvertProcessorType? Type { get; set; }
+
+		/// <summary>
+		/// If <c>true</c> and <see cref="Field" /> does not exist or is null,
+		/// the processor quietly exits without modifying the document. Default is <c>false</c>
+		/// </summary>
+		[JsonProperty("ignore_missing")]
+		bool? IgnoreMissing { get; set; }
 	}
 
 	public class ConvertProcessor : ProcessorBase, IConvertProcessor
 	{
+		/// <inheritdoc />
 		public Field Field { get; set; }
+		/// <inheritdoc />
 		public Field TargetField { get; set; }
+		/// <inheritdoc />
 		public ConvertProcessorType? Type { get; set; }
+		/// <inheritdoc />
+		public bool? IgnoreMissing { get; set; }
+		/// <inheritdoc />
 		protected override string Name => "convert";
 	}
 
@@ -34,19 +61,28 @@ public class ConvertProcessorDescriptor<T> : ProcessorDescriptorBase<ConvertProc
 		protected override string Name => "convert";
 		Field IConvertProcessor.Field { get; set; }
 		Field IConvertProcessor.TargetField { get; set; }
+		bool? IConvertProcessor.IgnoreMissing { get; set; }
 		ConvertProcessorType? IConvertProcessor.Type { get; set; }
 
+		/// <inheritdoc cref="IConvertProcessor.Field" />
 		public ConvertProcessorDescriptor<T> Field(Field field) => Assign(field, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IConvertProcessor.Field" />
 		public ConvertProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IConvertProcessor.TargetField" />
 		public ConvertProcessorDescriptor<T> TargetField(Field field) => Assign(field, (a, v) => a.TargetField = v);
 
+		/// <inheritdoc cref="IConvertProcessor.TargetField" />
 		public ConvertProcessorDescriptor<T> TargetField(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.TargetField = v);
 
+		/// <inheritdoc cref="IConvertProcessor.Type" />
 		public ConvertProcessorDescriptor<T> Type(ConvertProcessorType? type) => Assign(type, (a, v) => a.Type = v);
+
+		/// <inheritdoc cref="IConvertProcessor.IgnoreMissing" />
+		public ConvertProcessorDescriptor<T> IgnoreMissing(bool? ignoreMissing = true) => Assign(ignoreMissing, (a, v) => a.IgnoreMissing = v);
 	}
 
 	[JsonConverter(typeof(StringEnumConverter))]
@@ -55,9 +91,15 @@ public enum ConvertProcessorType
 		[EnumMember(Value = "integer")]
 		Integer,
 
+		[EnumMember(Value = "long")]
+		Long,
+
 		[EnumMember(Value = "float")]
 		Float,
 
+		[EnumMember(Value = "double")]
+		Double,
+
 		[EnumMember(Value = "string")]
 		String,
 

src/Nest/Ingest/Processors/DateProcessor.cs

@@ -9,33 +9,57 @@ namespace Nest
 	[JsonConverter(typeof(ProcessorJsonConverter<DateProcessor>))]
 	public interface IDateProcessor : IProcessor
 	{
+		/// <summary>
+		/// The field to get the date from.
+		/// </summary>
 		[JsonProperty("field")]
 		Field Field { get; set; }
 
+		/// <summary>
+		/// An array of the expected date formats. Can be a Joda pattern or one of
+		/// the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
+		/// </summary>
 		[JsonProperty("formats")]
 		IEnumerable<string> Formats { get; set; }
 
+		/// <summary>
+		/// The locale to use when parsing the date, relevant when parsing month names or week days.
+		/// Supports template snippets.
+		/// </summary>
 		[JsonProperty("locale")]
 		string Locale { get; set; }
 
+		/// <summary>
+		/// The field that will hold the parsed date. Defaults to @timestamp
+		/// </summary>
 		[JsonProperty("target_field")]
 		Field TargetField { get; set; }
 
+		/// <summary>
+		/// The timezone to use when parsing the date. Supports template snippets.
+		/// </summary>
 		[JsonProperty("timezone")]
 		string Timezone { get; set; }
 	}
 
 	public class DateProcessor : ProcessorBase, IDateProcessor
 	{
+		/// <inheritdoc />
 		public Field Field { get; set; }
 
+		/// <inheritdoc />
 		public IEnumerable<string> Formats { get; set; }
 
+		/// <inheritdoc />
 		public string Locale { get; set; }
 
+		/// <inheritdoc />
 		public Field TargetField { get; set; }
 
+		/// <inheritdoc />
 		public string Timezone { get; set; }
+
+		/// <inheritdoc />
 		protected override string Name => "date";
 	}
 
@@ -51,22 +75,30 @@ public class DateProcessorDescriptor<T>
 		Field IDateProcessor.TargetField { get; set; }
 		string IDateProcessor.Timezone { get; set; }
 
+		/// <inheritdoc cref="IDateProcessor.Field" />
 		public DateProcessorDescriptor<T> Field(Field field) => Assign(field, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IDateProcessor.Field" />
 		public DateProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IDateProcessor.TargetField" />
 		public DateProcessorDescriptor<T> TargetField(Field field) => Assign(field, (a, v) => a.TargetField = v);
 
+		/// <inheritdoc cref="IDateProcessor.TargetField" />
 		public DateProcessorDescriptor<T> TargetField(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.TargetField = v);
 
+		/// <inheritdoc cref="IDateProcessor.Formats" />
 		public DateProcessorDescriptor<T> Formats(IEnumerable<string> matchFormats) => Assign(matchFormats, (a, v) => a.Formats = v);
 
+		/// <inheritdoc cref="IDateProcessor.Formats" />
 		public DateProcessorDescriptor<T> Formats(params string[] matchFormats) => Assign(matchFormats, (a, v) => a.Formats = v);
 
+		/// <inheritdoc cref="IDateProcessor.Timezone" />
 		public DateProcessorDescriptor<T> Timezone(string timezone) => Assign(timezone, (a, v) => a.Timezone = v);
 
+		/// <inheritdoc cref="IDateProcessor.Locale" />
 		public DateProcessorDescriptor<T> Locale(string locale) => Assign(locale, (a, v) => a.Locale = v);
 	}
 }

src/Nest/Ingest/Processors/DissectProcessor.cs

@@ -22,8 +22,8 @@ public interface IDissectProcessor : IProcessor
 		[JsonProperty("pattern")]
 		string Pattern { get; set; }
 
-		/// <summary
-		/// If true and field does not exist or is null, the processor quietly exits without modifying the document>
+		/// <summary>
+		/// If <c>true</c> and field does not exist or is null, the processor quietly exits without modifying the document
 		/// </summary>
 		[JsonProperty("ignore_missing")]
 		bool? IgnoreMissing { get; set; }

src/Nest/Ingest/Processors/DotExpanderProcessor.cs

@@ -28,35 +28,21 @@ public interface IDotExpanderProcessor : IProcessor
 		string Path { get; set; }
 	}
 
-	/// <summary>
-	/// Expands a field with dots into an object field.
-	/// This processor allows fields with dots in the name to be accessible by other processors in the pipeline.
-	/// Otherwise these fields can’t be accessed by any processor.
-	/// </summary>
+	/// <inheritdoc cref="IDotExpanderProcessor" />
 	public class DotExpanderProcessor : ProcessorBase, IDotExpanderProcessor
 	{
-		/// <summary>
-		/// The field to expand into an object field
-		/// </summary>
+		/// <inheritdoc />
 		[JsonProperty("field")]
 		public Field Field { get; set; }
 
-		/// <summary>
-		/// The field that contains the field to expand.
-		/// Only required if the field to expand is part another object field,
-		/// because the field option can only understand leaf fields.
-		/// </summary>
+		/// <inheritdoc />
 		[JsonProperty("path")]
 		public string Path { get; set; }
 
 		protected override string Name => "dot_expander";
 	}
 
-	/// <summary>
-	/// Expands a field with dots into an object field.
-	/// This processor allows fields with dots in the name to be accessible by other processors in the pipeline.
-	/// Otherwise these fields can’t be accessed by any processor.
-	/// </summary>
+	/// <inheritdoc cref="IDotExpanderProcessor" />
 	public class DotExpanderProcessorDescriptor<T>
 		: ProcessorDescriptorBase<DotExpanderProcessorDescriptor<T>, IDotExpanderProcessor>, IDotExpanderProcessor
 		where T : class
@@ -66,22 +52,14 @@ public class DotExpanderProcessorDescriptor<T>
 		Field IDotExpanderProcessor.Field { get; set; }
 		string IDotExpanderProcessor.Path { get; set; }
 
-		/// <summary>
-		/// The field to expand into an object field
-		/// </summary>
+		/// <inheritdoc cref="IDotExpanderProcessor.Field" />
 		public DotExpanderProcessorDescriptor<T> Field(Field field) => Assign(field, (a, v) => a.Field = v);
 
-		/// <summary>
-		/// The field to expand into an object field
-		/// </summary>
+		/// <inheritdoc cref="IDotExpanderProcessor.Field" />
 		public DotExpanderProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.Field = v);
 
-		/// <summary>
-		/// The field that contains the field to expand.
-		/// Only required if the field to expand is part another object field,
-		/// because the field option can only understand leaf fields.
-		/// </summary>
+		/// <inheritdoc cref="IDotExpanderProcessor.Path" />
 		public DotExpanderProcessorDescriptor<T> Path(string path) => Assign(path, (a, v) => a.Path = v);
 	}
 }

src/Nest/Ingest/Processors/FailProcessor.cs

@@ -2,27 +2,38 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// Raises an exception. This is useful for when you expect a pipeline to
+	/// fail and want to relay a specific message to the requester.
+	/// </summary>
 	[JsonObject(MemberSerialization.OptIn)]
 	[JsonConverter(typeof(ProcessorJsonConverter<FailProcessor>))]
 	public interface IFailProcessor : IProcessor
 	{
+		/// <summary>
+		/// The error message thrown by the processor. Supports template snippets.
+		/// </summary>
 		[JsonProperty("message")]
 		string Message { get; set; }
 	}
 
+	/// <inheritdoc cref="IFailProcessor" />
 	public class FailProcessor : ProcessorBase, IFailProcessor
 	{
+		/// <inheritdoc />
 		public string Message { get; set; }
 		protected override string Name => "fail";
 	}
 
+	/// <inheritdoc cref="IFailProcessor" />
 	public class FailProcessorDescriptor
 		: ProcessorDescriptorBase<FailProcessorDescriptor, IFailProcessor>, IFailProcessor
 	{
 		protected override string Name => "fail";
 
 		string IFailProcessor.Message { get; set; }
 
+		/// <inheritdoc cref="IFailProcessor.Message" />
 		public FailProcessorDescriptor Message(string message) => Assign(message, (a, v) => a.Message = v);
 	}
 }

src/Nest/Ingest/Processors/ForeachProcessor.cs

@@ -6,40 +6,74 @@
 
 namespace Nest
 {
+	/// <summary>
+	/// Processes elements in an array of unknown length.
+	/// All processors can operate on elements inside an array, but if all elements of
+	/// an array need to be processed in the same way, defining a processor for each
+	/// element becomes cumbersome and tricky because it is likely that the number of
+	/// elements in an array is unknown. For this reason the foreach processor exists.
+	/// By specifying the field holding array elements and a processor that defines what
+	/// should happen to each element, array fields can easily be preprocessed.
+	/// </summary>
 	[JsonObject(MemberSerialization.OptIn)]
 	[JsonConverter(typeof(ProcessorJsonConverter<ForeachProcessor>))]
 	public interface IForeachProcessor : IProcessor
 	{
+		/// <summary>
+		/// The array field
+		/// </summary>
 		[JsonProperty("field")]
 		Field Field { get; set; }
 
+		/// <summary>
+		/// The processor to execute against each field
+		/// </summary>
 		[JsonProperty("processor")]
 		IProcessor Processor { get; set; }
+
+		/// <summary>
+		/// If <c>true</c> and <see cref="Field" /> does not exist or is null,
+		/// the processor quietly exits without modifying the document. Default is <c>false</c>
+		/// </summary>
+		[JsonProperty("ignore_missing")]
+		bool? IgnoreMissing { get; set; }
 	}
 
+	/// <inheritdoc cref="IForeachProcessor"/>
 	public class ForeachProcessor : ProcessorBase, IForeachProcessor
 	{
+		/// <inheritdoc />
 		public Field Field { get; set; }
+		/// <inheritdoc />
 		public IProcessor Processor { get; set; }
+		/// <inheritdoc />
+		public bool? IgnoreMissing { get; set; }
 		protected override string Name => "foreach";
 	}
 
+	/// <inheritdoc cref="IForeachProcessor"/>
 	public class ForeachProcessorDescriptor<T>
 		: ProcessorDescriptorBase<ForeachProcessorDescriptor<T>, IForeachProcessor>, IForeachProcessor
 		where T : class
 	{
 		protected override string Name => "foreach";
 
 		Field IForeachProcessor.Field { get; set; }
-
 		IProcessor IForeachProcessor.Processor { get; set; }
+		bool? IForeachProcessor.IgnoreMissing { get; set; }
 
+		/// <inheritdoc cref="IForeachProcessor.Field"/>
 		public ForeachProcessorDescriptor<T> Field(Field field) => Assign(field, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IForeachProcessor.Field"/>
 		public ForeachProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) =>
 			Assign(objectPath, (a, v) => a.Field = v);
 
+		/// <inheritdoc cref="IForeachProcessor.Processor"/>
 		public ForeachProcessorDescriptor<T> Processor(Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> selector) =>
 			Assign(selector, (a, v) => a.Processor = v?.Invoke(new ProcessorsDescriptor())?.Value?.FirstOrDefault());
+
+		/// <inheritdoc cref="IForeachProcessor.IgnoreMissing" />
+		public ForeachProcessorDescriptor<T> IgnoreMissing(bool? ignoreMissing = true) => Assign(ignoreMissing, (a, v) => a.IgnoreMissing = v);
 	}
 }