From d1b845ea4d2d9768bec84a33f3280cb9e988c7e0 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 30 Jan 2020 13:45:26 -0800 Subject: [PATCH 1/5] double fix --- src/Microsoft.ML.Data/Transforms/KeyToValue.cs | 2 +- .../Transforms/ValueToKeyMappingTransformer.cs | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.Data/Transforms/KeyToValue.cs b/src/Microsoft.ML.Data/Transforms/KeyToValue.cs index 058eda3bf0..c1863a32cf 100644 --- a/src/Microsoft.ML.Data/Transforms/KeyToValue.cs +++ b/src/Microsoft.ML.Data/Transforms/KeyToValue.cs @@ -520,7 +520,7 @@ public override bool SaveOnnx(OnnxContext ctx, string srcVariableName, string ds long[] values = Array.ConvertAll(_values.GetValues().ToArray(), item => Convert.ToInt64(item)); node.AddAttribute("values_int64s", values); } - else if (TypeOutput == NumberDataViewType.Single) + else if (TypeOutput == NumberDataViewType.Double || TypeOutput == NumberDataViewType.Single) { float[] values = Array.ConvertAll(_values.GetValues().ToArray(), item => Convert.ToSingle(item)); node.AddAttribute("values_floats", values); diff --git a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs index 2d064943b2..92fea35433 100644 --- a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs +++ b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs @@ -786,6 +786,7 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src OnnxNode node; long[] termIds; string opType = "LabelEncoder"; + OnnxNode castNode; var labelEncoderOutput = ctx.AddIntermediateVariable(_types[iinfo], "LabelEncoderOutput", true); if (info.TypeSrc.GetItemType().Equals(TextDataViewType.Instance)) @@ -800,6 +801,16 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src var terms = GetTermsAndIds(iinfo, out termIds); node.AddAttribute("keys_floats", terms); } + else if (info.TypeSrc.GetItemType().Equals(NumberDataViewType.Double)) + { + var castOutput = ctx.AddIntermediateVariable(null, "castOutput", true); + castNode = ctx.CreateNode("Cast", srcVariableName, castOutput, ctx.GetNodeName(opType), ""); + var t2 = InternalDataKindExtensions.ToInternalDataKind(DataKind.Single).ToType(); + castNode.AddAttribute("to", t2); + node = ctx.CreateNode(opType, castOutput, labelEncoderOutput, ctx.GetNodeName(opType)); + var terms = GetTermsAndIds(iinfo, out termIds); + node.AddAttribute("keys_floats", terms); + } else { // LabelEncoder-2 in ORT v1 only supports the following mappings @@ -822,7 +833,7 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src InternalDataKindExtensions.TryGetDataKind(_parent._unboundMaps[iinfo].OutputType.RawType, out dataKind); opType = "Cast"; - var castNode = ctx.CreateNode(opType, labelEncoderOutput, dstVariableName, ctx.GetNodeName(opType), ""); + castNode = ctx.CreateNode(opType, labelEncoderOutput, dstVariableName, ctx.GetNodeName(opType), ""); castNode.AddAttribute("to", dataKind.ToType()); return true; From ebfb1404fe48ceedda2aeeb386af93b8c113458c Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 30 Jan 2020 17:21:47 -0800 Subject: [PATCH 2/5] added support for int64 keys --- .../Transforms/ValueToKeyMappingTransformer.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs index 92fea35433..b72bb59012 100644 --- a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs +++ b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs @@ -811,6 +811,16 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src var terms = GetTermsAndIds(iinfo, out termIds); node.AddAttribute("keys_floats", terms); } + else if (info.TypeSrc.GetItemType().Equals(NumberDataViewType.Int64)) + { + var castOutput = ctx.AddIntermediateVariable(null, "castOutput", true); + castNode = ctx.CreateNode("Cast", srcVariableName, castOutput, ctx.GetNodeName(opType), ""); + var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.String).ToType(); + castNode.AddAttribute("to", t); + node = ctx.CreateNode(opType, castOutput, labelEncoderOutput, ctx.GetNodeName(opType)); + var terms = GetTermsAndIds(iinfo, out termIds); + node.AddAttribute("keys_strings", terms.Select(item => item.ToString())); + } else { // LabelEncoder-2 in ORT v1 only supports the following mappings From 083ede9a5864475d366f166053f60a9c61111e04 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 30 Jan 2020 17:23:46 -0800 Subject: [PATCH 3/5] number change --- .../Transforms/ValueToKeyMappingTransformer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs index b72bb59012..4f39d5af0b 100644 --- a/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs +++ b/src/Microsoft.ML.Data/Transforms/ValueToKeyMappingTransformer.cs @@ -805,8 +805,8 @@ private bool SaveAsOnnxCore(OnnxContext ctx, int iinfo, ColInfo info, string src { var castOutput = ctx.AddIntermediateVariable(null, "castOutput", true); castNode = ctx.CreateNode("Cast", srcVariableName, castOutput, ctx.GetNodeName(opType), ""); - var t2 = InternalDataKindExtensions.ToInternalDataKind(DataKind.Single).ToType(); - castNode.AddAttribute("to", t2); + var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.Single).ToType(); + castNode.AddAttribute("to", t); node = ctx.CreateNode(opType, castOutput, labelEncoderOutput, ctx.GetNodeName(opType)); var terms = GetTermsAndIds(iinfo, out termIds); node.AddAttribute("keys_floats", terms); From 81f9588a7e8066c6f340917f330ff62d22174d2a Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Fri, 31 Jan 2020 09:27:33 -0800 Subject: [PATCH 4/5] modifying test case --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index f138917912..723ccc3f81 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -1093,7 +1093,8 @@ public void IndicateMissingValuesOnnxConversionTest() } [Theory] - [InlineData(DataKind.Single)] + [InlineData(DataKind.Int64)] + [InlineData(DataKind.Double)] [InlineData(DataKind.String)] public void ValueToKeyMappingOnnxConversionTest(DataKind valueType) { From d27fa8c2a4804af8c5a28d0fba30a8225aad14b9 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Fri, 31 Jan 2020 09:30:08 -0800 Subject: [PATCH 5/5] modifying test case --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 723ccc3f81..e748d43e74 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -1093,6 +1093,7 @@ public void IndicateMissingValuesOnnxConversionTest() } [Theory] + [InlineData(DataKind.Single)] [InlineData(DataKind.Int64)] [InlineData(DataKind.Double)] [InlineData(DataKind.String)]