Backport of: Fix index creation issue when multiple options are used with fields (#405) (#413)

Fix index creation issue when multiple options are used with fields   (#405)

* - change order of field options in search&query
- add missing xml doc for parameters

* fix search tests

* fix format

Author: atakavci

src/NRedisStack/Search/Schema.cs

@@ -97,11 +97,10 @@ internal override void AddFieldTypeArgs(List<object> args)
                 AddPhonetic(args);
                 AddWeight(args);
                 if (WithSuffixTrie) args.Add(SearchArgs.WITHSUFFIXTRIE);
-                if (Sortable) args.Add(FieldOptions.SORTABLE);
                 if (Unf) args.Add(SearchArgs.UNF);
                 if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
                 if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY);
-
+                if (Sortable) args.Add(FieldOptions.SORTABLE);
             }
 
             private void AddWeight(List<object> args)
@@ -165,10 +164,10 @@ internal override void AddFieldTypeArgs(List<object> args)
                     args.Add(Separator);
                 }
                 if (CaseSensitive) args.Add(SearchArgs.CASESENSITIVE);
-                if (Sortable) args.Add(FieldOptions.SORTABLE);
                 if (Unf) args.Add(SearchArgs.UNF);
                 if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
                 if (EmptyIndex) args.Add(FieldOptions.INDEXEMPTY);
+                if (Sortable) args.Add(FieldOptions.SORTABLE);
             }
         }
 
@@ -192,10 +191,9 @@ internal GeoField(string name, bool sortable = false, bool noIndex = false, bool
             internal override void AddFieldTypeArgs(List<object> args)
             {
                 if (NoIndex) args.Add(SearchArgs.NOINDEX);
-                if (Sortable) args.Add(FieldOptions.SORTABLE);
                 if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
+                if (Sortable) args.Add(FieldOptions.SORTABLE);
             }
-
         }
 
         public class GeoShapeField : Field
@@ -252,10 +250,9 @@ internal NumericField(string name, bool sortable = false, bool noIndex = false,
             internal override void AddFieldTypeArgs(List<object> args)
             {
                 if (NoIndex) args.Add(SearchArgs.NOINDEX);
-                if (Sortable) args.Add(FieldOptions.SORTABLE);
                 if (MissingIndex) args.Add(FieldOptions.INDEXMISSING);
+                if (Sortable) args.Add(FieldOptions.SORTABLE);
             }
-
         }
 
         public class VectorField : Field
@@ -322,6 +319,10 @@ public Schema AddField(Field field)
         /// <param name="unf">Set this to true to prevent the indexer from sorting on the normalized form.
         /// Normalied form is the field sent to lower case with all diaretics removed</param>
         /// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
+        /// <param name="emptyIndex">  allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddTextField(string name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false,
                                    string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false)
@@ -342,6 +343,10 @@ public Schema AddTextField(string name, double weight = 1.0, bool sortable = fal
         /// <param name="unf">Set this to true to prevent the indexer from sorting on the normalized form.
         /// Normalied form is the field sent to lower case with all diaretics removed</param>
         /// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
+        /// <param name="emptyIndex">  allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable = false, bool unf = false, bool noStem = false,
                                    string? phonetic = null, bool noIndex = false, bool withSuffixTrie = false, bool missingIndex = false, bool emptyIndex = false)
@@ -355,6 +360,9 @@ public Schema AddTextField(FieldName name, double weight = 1.0, bool sortable =
         /// </summary>
         /// <param name="name">The field's name.</param>
         /// <param name="system">The coordinate system to use.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddGeoShapeField(string name, CoordinateSystem system, bool missingIndex = false)
         {
@@ -367,6 +375,9 @@ public Schema AddGeoShapeField(string name, CoordinateSystem system, bool missin
         /// </summary>
         /// <param name="name">The field's name.</param>
         /// <param name="system">The coordinate system to use.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddGeoShapeField(FieldName name, CoordinateSystem system, bool missingIndex = false)
         {
@@ -380,6 +391,9 @@ public Schema AddGeoShapeField(FieldName name, CoordinateSystem system, bool mis
         /// <param name="name">The field's name.</param>
         /// <param name="sortable">If true, the text field can be sorted.</param>
         /// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
         {
@@ -393,6 +407,9 @@ public Schema AddGeoField(FieldName name, bool sortable = false, bool noIndex =
         /// <param name="name">The field's name.</param>
         /// <param name="sortable">If true, the text field can be sorted.</param>
         /// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddGeoField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
         {
@@ -406,6 +423,9 @@ public Schema AddGeoField(string name, bool sortable = false, bool noIndex = fal
         /// <param name="name">The field's name.</param>
         /// <param name="sortable">If true, the text field can be sorted.</param>
         /// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddNumericField(FieldName name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
         {
@@ -419,6 +439,9 @@ public Schema AddNumericField(FieldName name, bool sortable = false, bool noInde
         /// <param name="name">The field's name.</param>
         /// <param name="sortable">If true, the text field can be sorted.</param>
         /// <param name="noIndex">Attributes can have the NOINDEX option, which means they will not be indexed.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddNumericField(string name, bool sortable = false, bool noIndex = false, bool missingIndex = false)
         {
@@ -437,6 +460,10 @@ public Schema AddNumericField(string name, bool sortable = false, bool noIndex =
         /// <param name="caseSensitive">If true, Keeps the original letter cases of the tags.</param>
         /// Normalied form is the field sent to lower case with all diaretics removed</param>
         /// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
+        /// <param name="emptyIndex">  allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddTagField(FieldName name, bool sortable = false, bool unf = false,
                               bool noIndex = false, string separator = ",",
@@ -457,6 +484,10 @@ public Schema AddTagField(FieldName name, bool sortable = false, bool unf = fals
         /// <param name="caseSensitive">If true, Keeps the original letter cases of the tags.</param>
         /// Normalied form is the field sent to lower case with all diaretics removed</param>
         /// <param name="withSuffixTrie">Keeps a suffix trie with all terms which match the suffix.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
+        /// <param name="emptyIndex">  allows you to index and search for empty strings. By default, empty strings are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddTagField(string name, bool sortable = false, bool unf = false,
                               bool noIndex = false, string separator = ",",
@@ -471,7 +502,10 @@ public Schema AddTagField(string name, bool sortable = false, bool unf = false,
         /// </summary>
         /// <param name="name">The field's name.</param>
         /// <param name="algorithm">The vector similarity algorithm to use.</param>
-        /// <param name="attribute">The algorithm attributes for the creation of the vector index.</param>
+        /// <param name="attributes">The algorithm attributes for the creation of the vector index.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary<string, object>? attributes = null, bool missingIndex = false)
         {
@@ -484,7 +518,10 @@ public Schema AddVectorField(FieldName name, VectorAlgo algorithm, Dictionary<st
         /// </summary>
         /// <param name="name">The field's name.</param>
         /// <param name="algorithm">The vector similarity algorithm to use.</param>
-        /// <param name="attribute">The algorithm attributes for the creation of the vector index.</param>
+        /// <param name="attributes">The algorithm attributes for the creation of the vector index.</param>
+        /// <param name="missingIndex"> search for missing values, that is, documents that do not contain a specific field. 
+        /// Note the difference between a field with an empty value and a document with a missing value. 
+        /// By default, missing values are not indexed.</param>
         /// <returns>The <see cref="Schema"/> object.</returns>
         public Schema AddVectorField(string name, VectorAlgo algorithm, Dictionary<string, object>? attributes = null, bool missingIndex = false)
         {

tests/NRedisStack.Tests/Search/IndexCreationTests.cs

@@ -11,6 +11,7 @@ public class IndexCreationTests : AbstractNRedisStackTest, IDisposable
     private readonly string index = "MISSING_EMPTY_INDEX";
     private static readonly string INDEXMISSING = "INDEXMISSING";
     private static readonly string INDEXEMPTY = "INDEXEMPTY";
+    private static readonly string SORTABLE = "SORTABLE";
 
     public IndexCreationTests(RedisFixture redisFixture) : base(redisFixture) { }
 
@@ -163,4 +164,82 @@ public void TestCreateFloat16VectorField()
         res = ft.Search("idx", q.AddParam("vec", vec2ToBytes));
         Assert.Equal(2, res.TotalResults);
     }
+
+    [Fact]
+    public void TestMissingSortableFieldCommandArgs()
+    {
+        string idx = "MISSING_EMPTY_SORTABLE_INDEX";
+        Schema sc = new Schema()
+                .AddTextField("text1", 1.0, missingIndex: true, emptyIndex: true, sortable: true)
+                .AddTagField("tag1", missingIndex: true, emptyIndex: true, sortable: true)
+                .AddNumericField("numeric1", missingIndex: true, sortable: true)
+                .AddGeoField("geo1", missingIndex: true, sortable: true);
+
+        var ftCreateParams = FTCreateParams.CreateParams();
+
+        var cmd = SearchCommandBuilder.Create(idx, ftCreateParams, sc);
+        var expectedArgs = new object[] { idx, "SCHEMA",
+                                            "text1","TEXT",INDEXMISSING,INDEXEMPTY,SORTABLE,
+                                            "tag1","TAG", INDEXMISSING,INDEXEMPTY,SORTABLE,
+                                            "numeric1","NUMERIC", INDEXMISSING,SORTABLE,
+                                            "geo1","GEO", INDEXMISSING, SORTABLE};
+        Assert.Equal(expectedArgs, cmd.Args);
+    }
+
+    [SkipIfRedis(Is.OSSCluster, Comparison.LessThan, "7.3.240")]
+    public void TestCombiningMissingEmptySortableFields()
+    {
+        string idx = "MISSING_EMPTY_SORTABLE_INDEX";
+        IDatabase db = redisFixture.Redis.GetDatabase();
+        var ft = db.FT(2);
+        var vectorAttrs = new Dictionary<string, object>()
+        {
+            ["TYPE"] = "FLOAT32",
+            ["DIM"] = "2",
+            ["DISTANCE_METRIC"] = "L2",
+        };
+        Schema sc = new Schema()
+           .AddTextField("text1", 1.0, missingIndex: true, emptyIndex: true, sortable: true)
+           .AddTagField("tag1", missingIndex: true, emptyIndex: true, sortable: true)
+           .AddNumericField("numeric1", missingIndex: true, sortable: true)
+           .AddGeoField("geo1", missingIndex: true, sortable: true)
+           .AddGeoShapeField("geoshape1", Schema.GeoShapeField.CoordinateSystem.FLAT, missingIndex: true)
+           .AddVectorField("vector1", Schema.VectorField.VectorAlgo.FLAT, vectorAttrs, missingIndex: true);
+
+        var ftCreateParams = FTCreateParams.CreateParams();
+        Assert.True(ft.Create(idx, ftCreateParams, sc));
+
+        var sampleHash = new HashEntry[] { new("field1", "value1"), new("field2", "value2") };
+        db.HashSet("hashWithMissingFields", sampleHash);
+
+        Polygon polygon = new GeometryFactory().CreatePolygon(new Coordinate[] { new Coordinate(1, 1), new Coordinate(10, 10), new Coordinate(100, 100), new Coordinate(1, 1), });
+
+        var hashWithAllFields = new HashEntry[] { new("text1", "value1"), new("tag1", "value2"), new("numeric1", "3.141"), new("geo1", "-0.441,51.458"), new("geoshape1", polygon.ToString()), new("vector1", "aaaaaaaa") };
+        db.HashSet("hashWithAllFields", hashWithAllFields);
+
+        var result = ft.Search(idx, new Query("ismissing(@text1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+
+        result = ft.Search(idx, new Query("ismissing(@tag1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+
+        result = ft.Search(idx, new Query("ismissing(@numeric1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+
+        result = ft.Search(idx, new Query("ismissing(@geo1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+
+        result = ft.Search(idx, new Query("ismissing(@geoshape1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+
+        result = ft.Search(idx, new Query("ismissing(@vector1)"));
+        Assert.Equal(1, result.TotalResults);
+        Assert.Equal("hashWithMissingFields", result.Documents[0].Id);
+    }
+
 }

tests/NRedisStack.Tests/Search/SearchTests.cs

@@ -2129,8 +2129,8 @@ public void TestFieldsCommandBuilder()
             "PHONETIC",
             "dm:en",
             "WITHSUFFIXTRIE",
-            "SORTABLE",
             "UNF",
+            "SORTABLE",
             "num",
             "NUMERIC",
             "NOINDEX",
@@ -2146,8 +2146,8 @@ public void TestFieldsCommandBuilder()
             "SEPARATOR",
             ";",
             "CASESENSITIVE",
-            "SORTABLE",
             "UNF",
+            "SORTABLE",
             "vec",
             "VECTOR",
             "FLAT",