Skip to content

Commit c30a63e

Browse files
LittleLittleCloudLittleLittleCloud
authored
add image featurizer to AutoFeaturizer (#6261)
1 parent a486a75 commit c30a63e

File tree

11 files changed

+319
-62
lines changed

11 files changed

+319
-62
lines changed

‎src/Microsoft.ML.AutoML/API/AutoCatalog.cs‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,44 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri
587587
return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) };
588588
}
589589

590+
internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName)
591+
{
592+
// load image => resize image (224, 224) => extract pixels => dnn featurizer
593+
var loadImageOption = new LoadImageOption
594+
{
595+
ImageFolder = null,
596+
InputColumnName = inputColumnName,
597+
OutputColumnName = outputColumnName,
598+
};
599+
600+
var resizeImageOption = new ResizeImageOption
601+
{
602+
ImageHeight = 224,
603+
ImageWidth = 224,
604+
InputColumnName = inputColumnName,
605+
OutputColumnName = outputColumnName,
606+
};
607+
608+
var extractPixelOption = new ExtractPixelsOption
609+
{
610+
InputColumnName = inputColumnName,
611+
OutputColumnName = outputColumnName,
612+
};
613+
614+
var dnnFeaturizerOption = new DnnFeaturizerImageOption
615+
{
616+
InputColumnName = inputColumnName,
617+
OutputColumnName = outputColumnName,
618+
};
619+
620+
var pipeline = new MultiModelPipeline();
621+
622+
return pipeline.Append(SweepableEstimatorFactory.CreateLoadImages(loadImageOption))
623+
.Append(SweepableEstimatorFactory.CreateResizeImages(resizeImageOption))
624+
.Append(SweepableEstimatorFactory.CreateExtractPixels(extractPixelOption))
625+
.Append(SweepableEstimatorFactory.CreateDnnFeaturizerImage(dnnFeaturizerOption));
626+
}
627+
590628
/// <summary>
591629
/// Create a single featurize pipeline according to <paramref name="data"/>. This function will collect all columns in <paramref name="data"/> and not in <paramref name="excludeColumns"/>,
592630
/// featurizing them using <see cref="CatalogFeaturizer(string[], string[])"/>, <see cref="NumericFeaturizer(string[], string[])"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
@@ -596,9 +634,10 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri
596634
/// <param name="catalogColumns">columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not.</param>
597635
/// <param name="numericColumns">columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not.</param>
598636
/// <param name="textColumns">columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not.</param>
637+
/// <param name="imagePathColumns">columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not.</param>
599638
/// <param name="outputColumnName">output feature column.</param>
600639
/// <param name="excludeColumns">columns that won't be included when featurizing, like label</param>
601-
public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] excludeColumns = null)
640+
public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null)
602641
{
603642
Contracts.CheckValue(data, nameof(data));
604643

@@ -646,6 +685,14 @@ public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "
646685
}
647686
}
648687

688+
if (imagePathColumns != null)
689+
{
690+
foreach (var column in imagePathColumns)
691+
{
692+
columnInfo.ImagePathColumnNames.Add(column);
693+
}
694+
}
695+
649696
return this.Featurizer(data, columnInfo, outputColumnName);
650697
}
651698

@@ -667,9 +714,11 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf
667714
var textFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.TextFeature);
668715
var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature);
669716
var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature);
717+
var imagePathFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.ImagePath);
670718
var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
671719
var numericFeatureColumnNames = numericFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
672720
var catalogFeatureColumnNames = catalogFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
721+
var imagePathColumnNames = imagePathFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
673722

674723
var pipeline = new MultiModelPipeline();
675724
if (numericFeatureColumnNames.Length > 0)
@@ -682,14 +731,19 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf
682731
pipeline = pipeline.Append(this.CatalogFeaturizer(catalogFeatureColumnNames, catalogFeatureColumnNames));
683732
}
684733

734+
foreach (var imagePathColumn in imagePathColumnNames)
735+
{
736+
pipeline = pipeline.Append(this.ImagePathFeaturizer(imagePathColumn, imagePathColumn));
737+
}
738+
685739
foreach (var textColumn in textFeatureColumnNames)
686740
{
687741
pipeline = pipeline.Append(this.TextFeaturizer(textColumn, textColumn));
688742
}
689743

690744
var option = new ConcatOption
691745
{
692-
InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).ToArray(),
746+
InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).Concat(imagePathColumnNames).ToArray(),
693747
OutputColumnName = outputColumnName,
694748
};
695749

‎src/Microsoft.ML.AutoML/CodeGen/dnn_featurizer_image_search_space.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"$schema": "./search-space-schema.json#",
3+
"name": "dnn_featurizer_image_option",
4+
"search_space": [
5+
{
6+
"name": "OutputColumnName",
7+
"type": "string"
8+
},
9+
{
10+
"name": "InputColumnName",
11+
"type": "string"
12+
},
13+
{
14+
"name": "ModelFactory",
15+
"type": "dnnModelFactory",
16+
"default": "resnet_18",
17+
"search_space": [
18+
"alexnet", "resnet_101", "resnet_18", "resnet_50"
19+
]
20+
}
21+
]
22+
}

‎src/Microsoft.ML.AutoML/CodeGen/estimator-schema.json‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
"ApplyOnnxModel",
6969
"ResizeImages",
7070
"ExtractPixels",
71+
"DnnFeaturizerImage",
7172
"Naive",
7273
"ForecastBySsa"
7374
]
@@ -180,22 +181,12 @@
180181
"confidenceLowerBoundColumn",
181182
"confidenceUpperBoundColumn",
182183
"confidenceLevel",
183-
"variableHorizon"
184+
"variableHorizon",
185+
"modelFactory"
184186
]
185187
},
186188
"argumentType": {
187-
"type": "string",
188-
"enum": [
189-
"integer",
190-
"float",
191-
"double",
192-
"string",
193-
"boolean",
194-
"resizingKind",
195-
"colorBits",
196-
"colorsOrder",
197-
"anchor"
198-
]
189+
"$ref": "search-space-schema.json#/definitions/option_type"
199190
}
200191
}
201192
}

‎src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,29 @@
22
"$schema": "http://json-schema.org/draft-04/schema",
33
"title": "Search Space",
44
"definitions": {
5+
"boolArray": {
6+
"type": "array",
7+
"items": { "type": "boolean" }
8+
},
9+
"intArray": {
10+
"type": "array",
11+
"items": { "type": "integer" }
12+
},
13+
"dnnModelFactoryArray": {
14+
"type": "array",
15+
"items": {
16+
"$ref": "#/definitions/dnnModelFactoryType"
17+
}
18+
},
19+
"dnnModelFactoryType": {
20+
"type": "string",
21+
"enum": [
22+
"resnet_18",
23+
"resnet_50",
24+
"resnet_101",
25+
"alexnet"
26+
]
27+
},
528
"range": {
629
"type": "object",
730
"properties": {
@@ -12,18 +35,17 @@
1235
"required": [ "min", "max" ]
1336
},
1437
"choice": {
15-
"type": "object",
16-
"properties": {
17-
"value": {
18-
"oneOf": [
19-
{ "type": "string" },
20-
{ "type": "number" },
21-
{ "type": "integer" },
22-
{ "type": "boolean" }
23-
]
38+
"oneOf": [
39+
{
40+
"$ref": "#/definitions/intArray"
41+
},
42+
{
43+
"$ref": "#/definitions/dnnModelFactoryArray"
44+
},
45+
{
46+
"$ref": "#/definitions/boolArray"
2447
}
25-
},
26-
"required": [ "value" ]
48+
]
2749
},
2850
"option": {
2951
"type": "object",
@@ -86,7 +108,8 @@
86108
"extract_pixels_option",
87109
"load_image_option",
88110
"image_classification_option",
89-
"matrix_factorization_option"
111+
"matrix_factorization_option",
112+
"dnn_featurizer_image_option"
90113
]
91114
},
92115
"option_name": {
@@ -130,7 +153,8 @@
130153
"ApproximationRank",
131154
"NumberOfIterations",
132155
"Quiet",
133-
"OutputAsFloatArray"
156+
"OutputAsFloatArray",
157+
"ModelFactory"
134158
]
135159
},
136160
"option_type": {
@@ -145,7 +169,8 @@
145169
"resizingKind",
146170
"colorBits",
147171
"colorsOrder",
148-
"anchor"
172+
"anchor",
173+
"dnnModelFactory"
149174
]
150175
}
151176
},

‎src/Microsoft.ML.AutoML/CodeGen/transformer-estimators.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,27 @@
321321
"nugetDependencies": [ "Microsoft.ML", "Microsoft.ML.ImageAnalytics" ],
322322
"usingStatements": [ "Microsoft.ML" ],
323323
"searchOption": "load_image_option"
324+
},
325+
{
326+
"functionName": "DnnFeaturizerImage",
327+
"estimatorTypes": [ "Transforms" ],
328+
"arguments": [
329+
{
330+
"argumentName": "outputColumnName",
331+
"argumentType": "string"
332+
},
333+
{
334+
"argumentName": "inputColumnName",
335+
"argumentType": "string"
336+
},
337+
{
338+
"argumentName": "modelFactory",
339+
"argumentType": "dnnModelFactory"
340+
}
341+
],
342+
"nugetDependencies": [ "Microsoft.ML.OnnxTransformer", "Microsoft.ML.OnnxRuntime" ],
343+
"usingStatements": [ "Microsoft.ML" ],
344+
"searchOption": "dnn_featurizer_image_option"
324345
}
325346
]
326347
}

‎src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@
3535
<PrivateAssets>all</PrivateAssets>
3636
</ProjectReference>
3737
<ProjectReference Include="..\Microsoft.ML.CpuMath\Microsoft.ML.CpuMath.csproj" />
38+
<ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.AlexNet\Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj" />
39+
<ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet101\Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj" />
40+
<ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj" />
41+
<ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet50\Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj" />
3842
<ProjectReference Include="..\Microsoft.ML.OnnxTransformer\Microsoft.ML.OnnxTransformer.csproj" />
3943
<ProjectReference Include="..\Microsoft.ML.SearchSpace\Microsoft.ML.SearchSpace.csproj">
4044
<PrivateAssets>all</PrivateAssets>

‎src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
4-
4+
using System;
55
namespace Microsoft.ML.AutoML.CodeGen
66
{
77
internal partial class LoadImages
@@ -44,4 +44,28 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Imag
4444
return context.MulticlassClassification.Trainers.ImageClassification(param.LabelColumnName, param.FeatureColumnName, param.ScoreColumnName);
4545
}
4646
}
47+
48+
internal partial class DnnFeaturizerImage
49+
{
50+
public override IEstimator<ITransformer> BuildFromOption(MLContext context, DnnFeaturizerImageOption param)
51+
{
52+
switch (param.ModelFactory)
53+
{
54+
case "resnet_50":
55+
return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
56+
m => m.ModelSelector.ResNet50(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
57+
case "resnet_18":
58+
return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
59+
m => m.ModelSelector.ResNet18(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
60+
case "resnet_101":
61+
return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
62+
m => m.ModelSelector.ResNet101(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
63+
case "alexnet":
64+
return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
65+
m => m.ModelSelector.AlexNet(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
66+
default:
67+
throw new NotImplementedException();
68+
}
69+
}
70+
}
4771
}

‎test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"schema": "e0 * e1 * e2 * e3 * e4",
3+
"estimators": {
4+
"e0": {
5+
"estimatorType": "LoadImages",
6+
"parameter": {
7+
"OutputColumnName": "ImagePath",
8+
"InputColumnName": "ImagePath"
9+
}
10+
},
11+
"e1": {
12+
"estimatorType": "ResizeImages",
13+
"parameter": {
14+
"OutputColumnName": "ImagePath",
15+
"InputColumnName": "ImagePath",
16+
"ImageHeight": 224,
17+
"ImageWidth": 224,
18+
"CropAnchor": "Center",
19+
"Resizing": "Fill"
20+
}
21+
},
22+
"e2": {
23+
"estimatorType": "ExtractPixels",
24+
"parameter": {
25+
"OutputColumnName": "ImagePath",
26+
"InputColumnName": "ImagePath",
27+
"ColorsToExtract": "Rgb",
28+
"OrderOfExtraction": "ARGB",
29+
"OutputAsFloatArray": true
30+
}
31+
},
32+
"e3": {
33+
"estimatorType": "DnnFeaturizerImage",
34+
"parameter": {
35+
"OutputColumnName": "ImagePath",
36+
"InputColumnName": "ImagePath",
37+
"ModelFactory": "resnet_18"
38+
}
39+
},
40+
"e4": {
41+
"estimatorType": "Concatenate",
42+
"parameter": {
43+
"InputColumnNames": [
44+
"ImagePath"
45+
],
46+
"OutputColumnName": "Features"
47+
}
48+
}
49+
}
50+
}

0 commit comments

Comments
 (0)