From afe910cef19078b80a6f7d52b404eb2ba9424dde Mon Sep 17 00:00:00 2001 From: zewditu Hailemariam Date: Wed, 24 Jan 2024 11:40:19 -0800 Subject: [PATCH 1/6] Add new type to key-value --- src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json | 6 ++++-- .../CodeGen/type_converter_search_space.json | 4 ++++ .../SweepableEstimator/Estimators/MapValueToKey.cs | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json index 0ccb7b1fcf..4753fb9b6c 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json +++ b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json @@ -238,7 +238,8 @@ "AnswerIndexStartColumnName", "predictedAnswerColumnName", "TopKAnswers", - "TargetType" + "TargetType", + "KeyData" ] }, "option_type": { @@ -257,7 +258,8 @@ "dnnModelFactory", "bertArchitecture", "imageClassificationArchType", - "dataKind" + "dataKind", + "dataView" ] } }, diff --git a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json index 38d860e9c4..b512e32fc0 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json @@ -14,6 +14,10 @@ "name": "TargetType", "type": "dataKind", "default": "DataKind.Single" + }, + { + "name": "KeyData", + "type": "dataView" } ] } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs index a48827478f..6a78eada47 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs @@ -16,7 +16,7 @@ internal partial class MapKeyToValue { public override IEstimator BuildFromOption(MLContext context, MapKeyToValueOption param) { - return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName); + return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName, param.KeyData); } } } From 42201e00e54d7e5ec6f3bacc21f48346343900cd Mon Sep 17 00:00:00 2001 From: zewditu Hailemariam Date: Wed, 24 Jan 2024 11:58:19 -0800 Subject: [PATCH 2/6] Make it IDataView --- src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json | 2 +- .../CodeGen/type_converter_search_space.json | 2 +- .../Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json index 4753fb9b6c..61aff16ab7 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json +++ b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json @@ -259,7 +259,7 @@ "bertArchitecture", "imageClassificationArchType", "dataKind", - "dataView" + "iDataView" ] } }, diff --git a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json index b512e32fc0..55c8047a92 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json @@ -17,7 +17,7 @@ }, { "name": "KeyData", - "type": "dataView" + "type": "iDataView" } ] } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs index 379d9e618e..17559507d7 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs @@ -57,6 +57,7 @@ public void Execute(GeneratorExecutionContext context) "bertArchitecture" => "BertArchitecture", "imageClassificationArchType" => "Microsoft.ML.Vision.ImageClassificationTrainer.Architecture", "dataKind" => "Microsoft.ML.Data.DataKind", + "iDataView" => "IDataView", _ => throw new ArgumentException("unknown type"), }; From b6173b09af5047241aaab2aae4973f10363a8a67 Mon Sep 17 00:00:00 2001 From: zewditu Hailemariam Date: Wed, 24 Jan 2024 12:09:10 -0800 Subject: [PATCH 3/6] clean --- src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json index 289d9dbaa7..5b6e7d07c3 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json +++ b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json @@ -240,7 +240,7 @@ "predictedAnswerColumnName", "TopKAnswers", "TargetType", - "PredictionColumnName" + "PredictionColumnName", "KeyData" ] }, From 11e90bc703151a8fa500e059ce65717cc7f0a67e Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 24 Jan 2024 12:15:43 -0800 Subject: [PATCH 4/6] add dataview support in source generator --- .../CodeGen/map_value_to_key_search_space.json | 4 ++++ .../SweepableEstimator/Estimators/MapValueToKey.cs | 4 ++-- .../SearchSpaceGenerator.cs | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/map_value_to_key_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/map_value_to_key_search_space.json index 1663a95c18..6e9390f141 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/map_value_to_key_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/map_value_to_key_search_space.json @@ -14,6 +14,10 @@ "name": "AddKeyValueAnnotationsAsText", "type": "boolean", "default": false + }, + { + "name": "KeyData", + "type": "dataView" } ] } diff --git a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs index 6a78eada47..decbd39109 100644 --- a/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs +++ b/src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/MapValueToKey.cs @@ -8,7 +8,7 @@ internal partial class MapValueToKey { public override IEstimator BuildFromOption(MLContext context, MapValueToKeyOption param) { - return context.Transforms.Conversion.MapValueToKey(param.OutputColumnName, param.InputColumnName, addKeyValueAnnotationsAsText: param.AddKeyValueAnnotationsAsText); + return context.Transforms.Conversion.MapValueToKey(param.OutputColumnName, param.InputColumnName, addKeyValueAnnotationsAsText: param.AddKeyValueAnnotationsAsText, keyData: param.KeyData); } } @@ -16,7 +16,7 @@ internal partial class MapKeyToValue { public override IEstimator BuildFromOption(MLContext context, MapKeyToValueOption param) { - return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName, param.KeyData); + return context.Transforms.Conversion.MapKeyToValue(param.OutputColumnName, param.InputColumnName); } } } diff --git a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs index 379d9e618e..85ed140019 100644 --- a/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs +++ b/tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs @@ -57,6 +57,7 @@ public void Execute(GeneratorExecutionContext context) "bertArchitecture" => "BertArchitecture", "imageClassificationArchType" => "Microsoft.ML.Vision.ImageClassificationTrainer.Architecture", "dataKind" => "Microsoft.ML.Data.DataKind", + "dataView" => "Microsoft.ML.IDataView", _ => throw new ArgumentException("unknown type"), }; @@ -76,6 +77,7 @@ public void Execute(GeneratorExecutionContext context) (_, "BertArchitecture") => defaultToken.GetValue(), (_, "Microsoft.ML.Vision.ImageClassificationTrainer.Architecture") => defaultToken.GetValue(), (_, "Microsoft.ML.Data.DataKind") => defaultToken.GetValue(), + (_, "Microsoft.ML.IDataView") => defaultToken.GetValue(), (_, _) => throw new ArgumentException("unknown"), }; From a1fc04cd7f8c57303a6f66018782a858bbde76a9 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 24 Jan 2024 12:27:51 -0800 Subject: [PATCH 5/6] Update type_converter_search_space.json --- .../CodeGen/type_converter_search_space.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json index 55c8047a92..b512e32fc0 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json +++ b/src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json @@ -17,7 +17,7 @@ }, { "name": "KeyData", - "type": "iDataView" + "type": "dataView" } ] } From 45716772403a05603f6b96a3d75ceb65f2611c49 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 24 Jan 2024 12:28:18 -0800 Subject: [PATCH 6/6] Update search-space-schema.json --- src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json index 5b6e7d07c3..c0512afad1 100644 --- a/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json +++ b/src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json @@ -261,7 +261,7 @@ "bertArchitecture", "imageClassificationArchType", "dataKind", - "iDataView" + "dataView" ] } },