Added support for resnet50 architecture for image classification (#4349)

* Added support for resnet50 architecture for image classification

Created a sample to show usage of resnet50 V2.
Tested against Tensorflow we can see there is a definitive discrepancy
in accuracy.

Training for ~136 epochs (TF uses steps rather than epochs) we are able
to achieve an accuracy of 78.7% in Tensorflow, compared to our accuracy
which was 54.6%. This discrepancy can be accounted for due to the way in
which TF adjusts the learning rate over time. We have already begun to
make those changes to our code, and will be added in a separate change.

* Added benchmark for image classification

Added a benchmark for image classification which uses resnet50 and the
small flowers dataset.

In adding the benchmark found that ModelSavePath was not wired, and due
to how benchmarks run I needed this, so wired that up as well.

Additionally removed the sample for resnet50 as it added no value

* fixed comments.

* address comments.

* fix build issue.

Author: bpstark

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs

@@ -66,7 +66,7 @@ public static void Example()
                 {
                     FeaturesColumnName = "Image",
                     LabelColumnName = "Label",
-                    // Just by changing/selecting InceptionV3/MobilenetV2 here instead of 
+                     // Just by changing/selecting InceptionV3/MobilenetV2/ResnetV250 here instead of 
                     // ResnetV2101 you can try a different architecture/
                     // pre-trained model. 
                     Arch = ImageClassificationEstimator.Architecture.ResnetV2101,

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs

@@ -64,7 +64,7 @@ public static void Example()
                 { 
                     FeaturesColumnName = "Image",
                     LabelColumnName = "Label",
-                    // Just by changing/selecting InceptionV3/MobilenetV2 here instead of 
+                    // Just by changing/selecting InceptionV3/MobilenetV2/ResnetV250 here instead of 
                     // ResnetV2101 you can try a different architecture/
                     // pre-trained model. 
                     Arch = ImageClassificationEstimator.Architecture.ResnetV2101,

src/Microsoft.ML.Dnn/DnnUtils.cs

@@ -309,6 +309,14 @@ internal static DnnModel LoadDnnModel(IHostEnvironment env, ImageClassificationE
                         client.DownloadFile(new Uri($"{baseGitPath}"), @"mobilenet_v2.meta");
                     }
                 }
+                else if (arch == ImageClassificationEstimator.Architecture.ResnetV250)
+                {
+                    var baseGitPath = @"https://tlcresources.blob.core.windows.net/image/ResNetV250TensorFlow/resnet_v2_50_299.meta";
+                    using (WebClient client = new WebClient())
+                    {
+                        client.DownloadFile(new Uri($"{baseGitPath}"), @"resnet_v2_50_299.meta");
+                    }
+                }
 
             }
 

src/Microsoft.ML.Dnn/ImageClassificationTransform.cs

@@ -132,7 +132,7 @@ private static ImageClassificationTransformer Create(IHostEnvironment env, Model
 
             return new ImageClassificationTransformer(env, DnnUtils.LoadTFSession(env, modelBytes), outputs, inputs,
                 addBatchDimensionInput, 1, labelColumn, checkpointName, arch,
-                scoreColumnName, predictedColumnName, learningRate, null, classCount, true, predictionTensorName,
+                scoreColumnName, predictedColumnName, learningRate, null, null, classCount, true, predictionTensorName,
                 softMaxTensorName, jpegDataTensorName, resizeTensorName, keyValueAnnotations);
 
         }
@@ -157,7 +157,7 @@ internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificatio
         internal ImageClassificationTransformer(IHostEnvironment env, ImageClassificationEstimator.Options options, DnnModel tensorFlowModel, IDataView input)
             : this(env, tensorFlowModel.Session, options.OutputColumns, options.InputColumns, null, options.BatchSize,
                   options.LabelColumnName, options.FinalModelPrefix, options.Arch, options.ScoreColumnName,
-                  options.PredictedLabelColumnName, options.LearningRate, input.Schema)
+                  options.PredictedLabelColumnName, options.LearningRate, options.ModelSavePath, input.Schema)
         {
             Contracts.CheckValue(env, nameof(env));
             env.CheckValue(options, nameof(options));
@@ -775,7 +775,7 @@ private static void GetModelInfo(IHostEnvironment env, ModelLoadContext ctx, out
         internal ImageClassificationTransformer(IHostEnvironment env, Session session, string[] outputColumnNames,
             string[] inputColumnNames,
             bool? addBatchDimensionInput, int batchSize, string labelColumnName, string finalModelPrefix, Architecture arch,
-            string scoreColumnName, string predictedLabelColumnName, float learningRate, DataViewSchema inputSchema, int? classCount = null, bool loadModel = false,
+            string scoreColumnName, string predictedLabelColumnName, float learningRate, string modelSavePath, DataViewSchema inputSchema, int? classCount = null, bool loadModel = false,
             string predictionTensorName = null, string softMaxTensorName = null, string jpegDataTensorName = null, string resizeTensorName = null, string[] labelAnnotations = null)
             : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ImageClassificationTransformer)))
 
@@ -813,7 +813,7 @@ internal ImageClassificationTransformer(IHostEnvironment env, Session session, s
             else
                 _classCount = classCount.Value;
 
-            _checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), finalModelPrefix + ModelLocation[arch]);
+            _checkpointPath = modelSavePath != null ? modelSavePath : Path.Combine(Directory.GetCurrentDirectory(), finalModelPrefix + ModelLocation[arch]);
 
             // Configure bottleneck tensor based on the model.
             if (arch == ImageClassificationEstimator.Architecture.ResnetV2101)
@@ -831,6 +831,11 @@ internal ImageClassificationTransformer(IHostEnvironment env, Session session, s
                 _bottleneckOperationName = "import/MobilenetV2/Logits/Squeeze";
                 _inputTensorName = "import/input";
             }
+            else if (arch == ImageClassificationEstimator.Architecture.ResnetV250)
+            {
+                _bottleneckOperationName = "resnet_v2_50/SpatialSqueeze";
+                _inputTensorName = "input";
+            }
 
             _outputs = new[] { scoreColumnName, predictedLabelColumnName };
 
@@ -1086,7 +1091,8 @@ public enum Architecture
         {
             ResnetV2101,
             InceptionV3,
-            MobilenetV2
+            MobilenetV2,
+            ResnetV250
         };
 
         /// <summary>
@@ -1096,7 +1102,8 @@ public enum Architecture
         {
             { Architecture.ResnetV2101, @"resnet_v2_101_299.meta" },
             { Architecture.InceptionV3, @"InceptionV3.meta" },
-            { Architecture.MobilenetV2, @"mobilenet_v2.meta" }
+            { Architecture.MobilenetV2, @"mobilenet_v2.meta" },
+            { Architecture.ResnetV250, @"resnet_v2_50_299.meta" }
         };
 
         /// <summary>
@@ -1106,7 +1113,8 @@ public enum Architecture
         {
             { Architecture.ResnetV2101, new Tuple<int, int>(299,299) },
             { Architecture.InceptionV3, new Tuple<int, int>(299,299) },
-            { Architecture.MobilenetV2, new Tuple<int, int>(224,224) }
+            { Architecture.MobilenetV2, new Tuple<int, int>(224,224) },
+            { Architecture.ResnetV250, new Tuple<int, int>(299,299) }
         };
 
         /// <summary>
@@ -1425,6 +1433,12 @@ public sealed class Options
             [Argument(ArgumentType.AtMostOnce, HelpText = "Callback to report metrics during training and validation phase.", SortOrder = 15)]
             public ImageClassificationMetricsCallback MetricsCallback = null;
 
+            /// <summary>
+            /// Indicates the path where the newly retrained model should be saved.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Indicates the path where the newly retrained model should be saved.", SortOrder = 15)]
+            public string ModelSavePath = null;
+
             /// <summary>
             /// Indicates to evaluate the model on train set after every epoch.
             /// </summary>

test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs

@@ -0,0 +1,267 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ML.Data;
+using Microsoft.ML.Transforms;
+using BenchmarkDotNet.Attributes;
+using static Microsoft.ML.DataOperationsCatalog;
+using System.Net.Http;
+using System.Diagnostics;
+
+namespace Microsoft.ML.Benchmarks
+{
+    [Config(typeof(TrainConfig))]
+    public class ImageClassificationBench
+    {
+        private string assetsPath;
+        private MLContext mlContext;
+        private IDataView trainDataset;
+        private IDataView testDataset;
+
+
+        [GlobalSetup]
+        public void SetupData()
+        {
+            mlContext = new MLContext(seed: 1);
+            /*
+             * Running in benchmarks causes to create a new temporary dir for each run
+             * However this dir is deleted while still running, as such need to get one
+             * level up to prevent issues with saving data.
+             */
+            string assetsRelativePath = @"../../../../assets";
+            assetsPath = GetAbsolutePath(assetsRelativePath);
+
+            var outputMlNetModelFilePath = Path.Combine(assetsPath, "outputs",
+                "imageClassifier.zip");
+
+
+            string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs",
+                "images");
+
+            //Download the image set and unzip
+            string finalImagesFolderName = DownloadImageSet(
+                imagesDownloadFolderPath);
+            string fullImagesetFolderPath = Path.Combine(
+                imagesDownloadFolderPath, finalImagesFolderName);
+
+            //Load all the original images info
+            IEnumerable<ImageData> images = LoadImagesFromDirectory(
+                folder: fullImagesetFolderPath, useFolderNameAsLabel: true);
+
+            IDataView shuffledFullImagesDataset = mlContext.Data.ShuffleRows(
+                mlContext.Data.LoadFromEnumerable(images));
+
+            shuffledFullImagesDataset = mlContext.Transforms.Conversion
+                    .MapValueToKey("Label")
+                .Append(mlContext.Transforms.LoadImages("Image",
+                            fullImagesetFolderPath, false, "ImagePath"))
+                .Fit(shuffledFullImagesDataset)
+                .Transform(shuffledFullImagesDataset);
+
+            // Split the data 90:10 into train and test sets, train and
+            // evaluate.
+            TrainTestData trainTestData = mlContext.Data.TrainTestSplit(
+                shuffledFullImagesDataset, testFraction: 0.1, seed: 1);
+
+            trainDataset = trainTestData.TrainSet;
+            testDataset = trainTestData.TestSet;
+
+        }
+
+        [Benchmark]
+        public TransformerChain<KeyToValueMappingTransformer> TrainResnetV250()
+        {
+            var options = new ImageClassificationEstimator.Options()
+            {
+                FeaturesColumnName = "Image",
+                LabelColumnName = "Label",
+                Arch = ImageClassificationEstimator.Architecture.ResnetV250,
+                Epoch = 50,
+                BatchSize = 10,
+                LearningRate = 0.01f,
+                EarlyStoppingCriteria = new ImageClassificationEstimator.EarlyStopping(minDelta: 0.001f, patience: 20, metric: ImageClassificationEstimator.EarlyStoppingMetric.Loss),
+                ValidationSet = testDataset,
+                ModelSavePath = assetsPath,
+                DisableEarlyStopping = true
+            };
+            var pipeline = mlContext.Model.ImageClassification(options)
+            .Append(mlContext.Transforms.Conversion.MapKeyToValue(
+                outputColumnName: "PredictedLabel",
+                inputColumnName: "PredictedLabel"));
+
+            return pipeline.Fit(trainDataset);
+        }
+
+
+        public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder,
+            bool useFolderNameAsLabel = true)
+        {
+            var files = Directory.GetFiles(folder, "*",
+                searchOption: SearchOption.AllDirectories);
+            foreach (var file in files)
+            {
+                if (Path.GetExtension(file) != ".jpg" &&
+                    Path.GetExtension(file) != ".JPEG" &&
+                    Path.GetExtension(file) != ".png")
+                    continue;
+
+                var label = Path.GetFileName(file);
+                if (useFolderNameAsLabel)
+                    label = Directory.GetParent(file).Name;
+                else
+                {
+                    for (int index = 0; index < label.Length; index++)
+                    {
+                        if (!char.IsLetter(label[index]))
+                        {
+                            label = label.Substring(0, index);
+                            break;
+                        }
+                    }
+                }
+
+                yield return new ImageData()
+                {
+                    ImagePath = file,
+                    Label = label
+                };
+
+            }
+        }
+
+        public static string DownloadImageSet(string imagesDownloadFolder)
+        {
+            // get a set of images to teach the network about the new classes
+
+            //SINGLE SMALL FLOWERS IMAGESET (200 files)
+            string fileName = "flower_photos_small_set.zip";
+            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
+                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
+                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
+                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+
+            Download(url, imagesDownloadFolder, fileName);
+            UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
+
+            return Path.GetFileNameWithoutExtension(fileName);
+
+        }
+
+        public static bool Download(string url, string destDir, string destFileName)
+        {
+            if (destFileName == null)
+                destFileName = url.Split(Path.DirectorySeparatorChar).Last();
+
+            string relativeFilePath = Path.Combine(destDir, destFileName);
+
+
+            using (HttpClient client = new HttpClient())
+            {
+                if (File.Exists(relativeFilePath))
+                {
+                    var headerResponse = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead).Result;
+                    var totalSizeInBytes = headerResponse.Content.Headers.ContentLength;
+                    var currentSize = new FileInfo(relativeFilePath).Length;
+
+                    //If current file size is not equal to expected file size, re-download file
+                    if (currentSize != totalSizeInBytes)
+                    {
+                        File.Delete(relativeFilePath);
+                        var response = client.GetAsync(url).Result;
+                        using FileStream fileStream = new FileStream(relativeFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
+                        using Stream contentStream = response.Content.ReadAsStreamAsync().Result;
+                        contentStream.CopyTo(fileStream);
+                    }
+                }
+                else
+                {
+                    Directory.CreateDirectory(destDir);
+                    var response = client.GetAsync(url).Result;
+                    using FileStream fileStream = new FileStream(relativeFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
+                    using Stream contentStream = response.Content.ReadAsStreamAsync().Result;
+                    contentStream.CopyTo(fileStream);
+                }
+            }
+            return true;
+        }
+
+
+        public static void UnZip(String gzArchiveName, String destFolder)
+        {
+            var flag = gzArchiveName.Split(Path.DirectorySeparatorChar)
+                .Last()
+                .Split('.')
+                .First() + ".bin";
+
+            if (File.Exists(Path.Combine(destFolder, flag))) return;
+
+            ZipFile.ExtractToDirectory(gzArchiveName, destFolder);
+
+            File.Create(Path.Combine(destFolder, flag));
+            Console.WriteLine("");
+            Console.WriteLine("Extracting is completed.");
+        }
+
+        public static string GetAbsolutePath(string relativePath)
+        {
+            FileInfo _dataRoot = new FileInfo(typeof(
+                ImageClassificationBench).Assembly.Location);
+
+            string assemblyFolderPath = _dataRoot.Directory.FullName;
+
+            string fullPath = Path.Combine(assemblyFolderPath, relativePath);
+
+            return fullPath;
+        }
+
+        public class ImageData
+        {
+            [LoadColumn(0)]
+            public string ImagePath;
+
+            [LoadColumn(1)]
+            public string Label;
+        }
+
+    }
+    public static class HttpContentExtensions
+    {
+        public static Task ReadAsFileAsync(this HttpContent content, string filename, bool overwrite)
+        {
+            string pathname = Path.GetFullPath(filename);
+            if (!overwrite && File.Exists(filename))
+            {
+                throw new InvalidOperationException(string.Format("File {0} already exists.", pathname));
+            }
+
+            FileStream fileStream = null;
+            try
+            {
+                fileStream = new FileStream(pathname, FileMode.Create, FileAccess.Write, FileShare.None);
+                return content.CopyToAsync(fileStream).ContinueWith(
+                    (copyTask) =>
+                    {
+                        fileStream.Close();
+                    });
+            }
+            catch
+            {
+                if (fileStream != null)
+                {
+                    fileStream.Close();
+                }
+
+                throw;
+            }
+        }
+    }
+}