V1.5.6.2. Bug fixes, perf improvements, and more CUDA support

Author: ChrisMaunder

.vscode/tasks.json

@@ -88,13 +88,28 @@
             "problemMatcher": "$msCompile"
         },   
 
+        {
+            "label": "build-tests",      // Builds ONLY the .NET unit tests
+            "type": "process",
+            "group": "build",
+            "command": "dotnet",
+            "args": [
+                "build",
+                "${workspaceFolder}/tests/QueueServiceTests",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },   
+
         {
             "label": "build-all-linux",         // Builds all projects that can be built for Linux (ie not any Windows Forms apps)
             "group": "build",
             "dependsOrder": "sequence",
             "dependsOn": [
                 "build-server",
                 "build-analysisservices",
+				"build-tests"
             ]
         },
 
@@ -105,7 +120,8 @@
             "dependsOn": [
                 "build-server",
                 "build-analysisservices",
-                "build-explorer"
+                "build-explorer",
+				"build-tests"
             ]
         },
 

.wslconfig

@@ -0,0 +1,11 @@
+# Place this into /users/<username> 
+
+# Settings apply across all Linux distros running on WSL 2
+[wsl2]
+
+# Limits VM memory to use no more than 12 GB, this can be set as whole numbers using GB or MB
+# The default is 50% of available RAM and 8GB isn't (currently) enough for CodeProject AI Server GPU
+memory=12GB 
+
+# Sets amount of swap storage space to 8GB, default is 25% of available RAM
+swap=8GB

CodeProject.AI.sln

@@ -24,6 +24,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AnalysisLayer", "AnalysisLa
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Installers", "Installers", "{D885EE64-C1BD-44D6-84D8-1E46806298D9}"
+	ProjectSection(SolutionItems) = preProject
+		Installers\install_CUDnn.bat = Installers\install_CUDnn.bat
+	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Javascript", "Javascript", "{3A860CDD-94B9-4002-BA08-87E8822DDE50}"
 	ProjectSection(SolutionItems) = preProject
@@ -64,7 +67,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assets", "assets", "{CB26AB
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{460DB5C8-46F3-4407-A2DF-D9063D14493A}"
 	ProjectSection(SolutionItems) = preProject
-		commands.json = commands.json
 		src\endtimer.bat = src\endtimer.bat
 		src\kill_dotnet.bat = src\kill_dotnet.bat
 		src\kill_dotnet.sh = src\kill_dotnet.sh
@@ -193,7 +195,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Objects", "Objects", "{4ED5
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ObjectDetectionNet", "src\AnalysisLayer\ObjectDetectionNet\ObjectDetectionNet.csproj", "{25D75AFE-BEC9-43BF-BE44-B9068FF8E395}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodeProject.SenseAI.API.Server.Backend.Tests", "tests\QueueServiceTests\CodeProject.SenseAI.API.Server.Backend.Tests.csproj", "{F94FBD1C-02FB-4169-8C26-2D52234D4311}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodeProject.AI.API.Server.Backend.Tests", "tests\QueueServiceTests\CodeProject.AI.API.Server.Backend.Tests.csproj", "{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -299,14 +301,14 @@ Global
 		{25D75AFE-BEC9-43BF-BE44-B9068FF8E395}.Release|Any CPU.Build.0 = Release|Any CPU
 		{25D75AFE-BEC9-43BF-BE44-B9068FF8E395}.Release|x86.ActiveCfg = Release|Any CPU
 		{25D75AFE-BEC9-43BF-BE44-B9068FF8E395}.Release|x86.Build.0 = Release|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Debug|x86.Build.0 = Debug|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Release|x86.ActiveCfg = Release|Any CPU
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311}.Release|x86.Build.0 = Release|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Debug|x86.Build.0 = Debug|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Release|x86.ActiveCfg = Release|Any CPU
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -346,7 +348,7 @@ Global
 		{C2EFFA0A-E8EA-4AFE-8599-FC28CB7864FB} = {B10B59B5-9F63-41C2-BFBB-6C7311DC4E99}
 		{4ED567B5-C28D-48BB-AEDC-864E2B2C7204} = {B10B59B5-9F63-41C2-BFBB-6C7311DC4E99}
 		{25D75AFE-BEC9-43BF-BE44-B9068FF8E395} = {156BFEDA-D477-43B2-92DA-FCC9BAF1F893}
-		{F94FBD1C-02FB-4169-8C26-2D52234D4311} = {D982BD8C-2257-413B-8513-8043AB3035F3}
+		{031F17E0-BE84-42AF-B9FE-4F928CB03D1B} = {D982BD8C-2257-413B-8513-8043AB3035F3}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {83740BD9-AEEF-49C7-A722-D7703D3A38CB}

Installers/Dev/setup_dev_env_linux.sh

@@ -56,7 +56,7 @@ useColor="true"
 enableGPU="false"
 
 # are we ready to support CUDA enabled GPUs?
-ssupportCUDA="false"
+supportCUDA="false"
 
 # verbosity can be: quiet | info | loud
 verbosity="quiet"

README.md

@@ -20,6 +20,7 @@ transfer, and is easy to use.
 
 </div>
 
+
 # Why
 
 1. AI programming is something every single developer should be aware of. We wanted a fun project we could use to help teach developers and get them involved in AI. We'll be using CodeProject.AI as a focus for articles and exploration to make it fun and painless to learn AI programming.
@@ -90,7 +91,7 @@ It can detect stuff!
 
 CodeProject.AI can currently
 
-- Detect objects in images
+- Detect objects in images, including using custom models
 - Detect faces in images
 - Detect the type of scene represented in an image
 - Recognise faces that have been registered with the service
@@ -112,13 +113,11 @@ We will be constantly expanding the feature list.
 
 #### Supported Development Environments
 
-
 This current release works in Visual Studio 2019+ on Windows 10+, and Visual Studio Code on Windows 10+. Ubuntu and macOS (both Intel and Apple Silicon). 
 
 The current release supports CPU on each platform, as well as nVidia CUDA GPUs on Windows. Future releases will expand GPU support to Docker and other cards.
 
 
-
 ## How to Guides
 
  - [Installing CodeProject.AI on your machine](https://www.codeproject.com/ai/docs/why/install_on_windows.html). For those who have CodeProject.AI integrated with Home Assist or Blue Iris

THIRD-PARTY-NOTICES.md

@@ -352,6 +352,7 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 
+
 ### Swashbuckle.AspNetCore
 ##### Generate beautiful API documentation
 https://github.com/domaindrivendev/Swashbuckle.AspNetCore

src/API/Server/FrontEnd/BackendProcessRunner.cs

@@ -265,7 +265,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 loggerIsValid = false;
             }
 
-            bool launchAnalysisServices = _config.GetValue("LaunchAnalysisServices", true);
+            bool launchAnalysisServices          = _config.GetValue("LaunchAnalysisServices", true);
+            int  launchAnalysisServicesDelaySecs = _config.GetValue("LaunchAnalysisServicesDelaySecs", 3);
 
             // Setup routes.  Do this first so they are active during debug without launching services.
             foreach (var entry in _modules!)
@@ -310,7 +311,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
             // Let's make sure the front end is up and running before we start the backend 
             // analysis services
-            await Task.Delay(TimeSpan.FromSeconds(3), stoppingToken);
+            await Task.Delay(TimeSpan.FromSeconds(launchAnalysisServicesDelaySecs), stoppingToken);
 
             if (loggerIsValid)
             {
@@ -337,7 +338,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                     continue;
 
                 if (status.Status == ProcessStatusType.NotEnabled)
-                    _logger.LogWarning($"Not starting {module.Name} (Not set as enabled)");
+                {
+                    // _logger.LogWarning($"Not starting {module.Name} (Not set as enabled)");
+                }
+                else
+                {
+                    _logger.LogWarning($"Attempting to start {module.Name}, Runtime: {module.Runtime}, FilePath: {module.FilePath}");
+                }
 
                 if (status.Status == ProcessStatusType.Enabled && !string.IsNullOrEmpty(module.FilePath))
                 {
@@ -348,11 +355,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                         if (!string.IsNullOrWhiteSpace(routeInfo.Queue))
                             _queueServices.EnsureQueueExists(routeInfo.Queue);
 
-                    ProcessStartInfo procStartInfo = CreateProcessStartInfo(module, moduleId);
-
-                    // Start the process
                     try
                     {
+                        // Start the process
+                        ProcessStartInfo procStartInfo = CreateProcessStartInfo(module, moduleId);
+
                         if (loggerIsValid)
                             _logger.LogTrace($"Starting {ShrinkPath(procStartInfo.FileName, 50)} {ShrinkPath(procStartInfo.Arguments, 50)}");
 
@@ -404,7 +411,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                             if (string.IsNullOrEmpty(error))
                                 error = "No error provided";
 
-                            if (error != "info: Microsoft.Hosting.Lifetime[0]")
+                            if (error.Contains("LoadLibrary failed with error 126") &&
+                                error.Contains("onnxruntime_providers_cuda.dll"))
+                            {
+                                error = "Attempted to load ONNX runtime CUDA provider. No luck, moving on...";
+                                _logger.LogInformation(filename + error);
+                            }
+                            else if (error != "info: Microsoft.Hosting.Lifetime[0]")
                             {
                                 // TOTAL HACK. ONNX/Tensorflow output is WAY too verbose for an error
                                 if (error.Contains("I tensorflow/cc/saved_model/reader.cc:") ||
@@ -415,6 +428,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                             }
                         };
 
+                        status.Status = ProcessStatusType.Starting;
                         if (!process.Start())
                         {
                             process = null;
@@ -423,15 +437,20 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
                         if (process is not null)
                         {
+                            status.Started = DateTime.UtcNow;
                             process.BeginOutputReadLine();
                             process.BeginErrorReadLine();
 
                             if (loggerIsValid)
                                 _logger.LogInformation($"Started {module.Name} backend");
 
                             _runningProcesses.Add(process);
-                            status.Status = ProcessStatusType.Started;
-                            status.Started = DateTime.UtcNow;
+
+                            int postStartPauseSecs = 5; // module.PostStartPauseSecs ?? 5;
+
+                            // Trying to reduce startup CPU and Memory for Docker
+                            await Task.Delay(TimeSpan.FromSeconds(postStartPauseSecs));
+                            status.Status  = ProcessStatusType.Started;
                         }
                         else
                         {
@@ -446,6 +465,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                         if (loggerIsValid)
                         {
                             _logger.LogError(ex, $"Error trying to start { module.Name} ({module.FilePath})");
+                            _logger.LogError(ex.Message);
+                            _logger.LogError(ex.StackTrace);
                         }
 #if DEBUG
                         _logger.LogError($" *** Did you setup the Development environment?");
@@ -485,8 +506,16 @@ private ProcessStartInfo CreateProcessStartInfo(ModuleConfig module, string modu
             }
 
             // Setup the process we're going to launch
+#if Windows
+            // Windows paths can have spaces so need quotes
+            var executableName = $"\"{filePath}\"";
+#else
+            // the I'm assuming the directories don't have spaces in Linux and MacOS
+            // because the Process.Start is choking on the quotes
+            var executableName = filePath;
+#endif
             ProcessStartInfo? procStartInfo = (command == "execute" || command == "launcher")
-                ? new ProcessStartInfo($"\"{filePath}\"")
+                ? new ProcessStartInfo(executableName)
                 {
                     UseShellExecute        = false,
                     WorkingDirectory       = workingDirectory,
@@ -563,8 +592,8 @@ private bool IsEnabled(ModuleConfig module)
                 return _frontendOptions.PYTHON_PATH?.Replace(PythonRuntimeMarker, launcher);
             }
 
-            if (runtime == "dotnet")
-                return "dotnet";
+            if (runtime == "dotnet" || runtime == "execute" || runtime == "launcher")
+                return runtime;
 
             return null;
         }

src/API/Server/FrontEnd/Config/ModuleCollection.cs

@@ -53,6 +53,14 @@ public class ModuleConfig
         /// </summary>
         public string? Command { get; set; }
 
+        /*
+        /// <summary>
+        /// Gets or sets the number of seconds this module should pause after starting to ensure 
+        /// any resources that require startup (eg GPUs) are fully activated before moving on.
+        /// </summary>
+        public int? PostStartPauseSecs { get; set; }
+        */
+
         /// <summary>
         /// Gets or sets the path to the startup file relative to the module directory.
         /// </summary>
@@ -96,40 +104,6 @@ public class ModuleConfig
         public string[] Platforms { get; set; } = Array.Empty<string>();
 
         /*
-        /// <summary>
-        /// Gets or sets the time this module was started.
-        /// </summary>
-        // TODO: Move the status info to another class that references or includes
-        //  the module coniguration, if needed.
-        public DateTime? Started { get; set; } = null;
-
-        /// <summary>
-        /// Gets or sets the latest time a request from this module was spotted by the queue manager.
-        /// </summary>
-        // TODO: Move the status info to another class that references or includes
-        //  the module coniguration, if needed.
-        public DateTime? LastSeen { get; set; } = null;
-
-        /// <summary>
-        /// Gets a value indicating whether this process is currently active
-        /// </summary>
-        // TODO: Move the status info to another class that references or includes
-        //  the module coniguration, if needed.
-        public bool Running
-        {
-            get
-            {
-                return LastSeen != null && (DateTime.UtcNow - LastSeen!) < TimeSpan.FromSeconds(65);
-            }
-        }
-
-        /// <summary>
-        /// Gets or sets the number of requests processed
-        /// </summary>
-        // TODO: Move the status info to another class that references or includes
-        //  the module coniguration, if needed.
-        public int? Processed { get; set; } = 0;
-
         /// <summary>
         /// Gets or sets the name of the hardware acceleration provider.
         /// </summary>

src/API/Server/FrontEnd/Controllers/LogController.cs

@@ -51,6 +51,14 @@ public ResponseBase AddLog([FromForm] string? entry,
                 msg += "[[" + category + "]]";
             if (!string.IsNullOrWhiteSpace(label))
                 msg += "{{" + label + "}}";
+
+            if (entry.Contains("LoadLibrary failed with error 126") &&
+                entry.Contains("onnxruntime_providers_cuda.dll"))
+            {
+                entry = "Attempted to load ONNX runtime CUDA provider. No luck, moving on...";
+                log_level = LogLevel.Information;
+            }
+
             msg += entry;
 
             switch (log_level)

src/API/Server/FrontEnd/Controllers/StatusController.cs

@@ -163,7 +163,10 @@ public ResponseBase ListAnalysisStatus()
             // List them out and return the status
             var response = new AnalysisServicesStatusResponse
             {
-                statuses = backend.ProcessStatuses.Values.ToList()
+                statuses = backend.ProcessStatuses
+                                  .Values
+                                  .Where(module => module.Status != ProcessStatusType.NotEnabled)
+                                  .ToList()
             };
 
             return response;