Merge pull request #38 from Martin-Molinero/initialization-performance-hang-fixes

Initialization hang fix

Author: jaredbroad

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.5.26
+current_version = 1.0.5.28
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?
 serialize = 
 	{major}.{minor}.{patch}.{release}{dev}

conda.recipe/meta.yaml

@@ -1,6 +1,6 @@
 package:
   name: pythonnet
-  version: "1.0.5.26"
+  version: "1.0.5.28"
 
 build:
   skip: True  # [not win]

setup.py

@@ -485,7 +485,7 @@ def run(self):
 
 setup(
     name="pythonnet",
-    version="1.0.5.26",
+    version="1.0.5.28",
     description=".Net and Mono integration for Python",
     url='https://pythonnet.github.io/',
     license='MIT',

src/SharedAssemblyInfo.cs

@@ -25,4 +25,4 @@
 // Version Information. Keeping it simple. May need to revisit for Nuget
 // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/
 // AssemblyVersion can only be numeric
-[assembly: AssemblyVersion("1.0.5.26")]
+[assembly: AssemblyVersion("1.0.5.28")]

src/clrmodule/ClrModule.cs

@@ -53,7 +53,7 @@ public static void initclr()
         {
 #if USE_PYTHON_RUNTIME_VERSION
             // Has no effect until SNK works. Keep updated anyways.
-            Version = new Version("1.0.5.26"),
+            Version = new Version("1.0.5.28"),
 #endif
             CultureInfo = CultureInfo.InvariantCulture
         };

src/runtime/assemblymanager.cs

@@ -1,11 +1,11 @@
 using System;
-using System.Collections;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Reflection;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace Python.Runtime
 {
@@ -17,17 +17,15 @@ internal class AssemblyManager
     {
         // modified from event handlers below, potentially triggered from different .NET threads
         // therefore this should be a ConcurrentDictionary
-        private static ConcurrentDictionary<string, ConcurrentDictionary<Assembly, string>> namespaces;
-        //private static Dictionary<string, Dictionary<string, string>> generics;
-        private static AssemblyLoadEventHandler lhandler;
-        private static ResolveEventHandler rhandler;
+        private static ConcurrentDictionary<string, ConcurrentDictionary<Assembly, byte>> namespaces;
+        private static ConcurrentDictionary<string, Assembly> assembliesNamesCache;
+        private static ConcurrentDictionary<string, Type> lookupTypeCache;
+        private static ConcurrentQueue<Assembly> assemblies;
+        private static int pendingAssemblies;
 
         // updated only under GIL?
         private static Dictionary<string, int> probed;
-
-        // modified from event handlers below, potentially triggered from different .NET threads
-        private static ConcurrentQueue<Assembly> assemblies;
-        internal static List<string> pypath;
+        private static List<string> pypath;
 
         private AssemblyManager()
         {
@@ -40,33 +38,41 @@ private AssemblyManager()
         /// </summary>
         internal static void Initialize()
         {
-            namespaces = new ConcurrentDictionary<string, ConcurrentDictionary<Assembly, string>>();
+            namespaces = new ConcurrentDictionary<string, ConcurrentDictionary<Assembly, byte>>();
+            assembliesNamesCache = new ConcurrentDictionary<string, Assembly>();
+            lookupTypeCache = new ConcurrentDictionary<string, Type>();
             probed = new Dictionary<string, int>(32);
-            //generics = new Dictionary<string, Dictionary<string, string>>();
             assemblies = new ConcurrentQueue<Assembly>();
             pypath = new List<string>(16);
 
             AppDomain domain = AppDomain.CurrentDomain;
 
-            lhandler = new AssemblyLoadEventHandler(AssemblyLoadHandler);
-            domain.AssemblyLoad += lhandler;
-
-            rhandler = new ResolveEventHandler(ResolveHandler);
-            domain.AssemblyResolve += rhandler;
+            domain.AssemblyLoad += AssemblyLoadHandler;
+            domain.AssemblyResolve += ResolveHandler;
 
-            Assembly[] items = domain.GetAssemblies();
-            foreach (Assembly a in items)
+            foreach (var assembly in domain.GetAssemblies())
             {
                 try
                 {
-                    ScanAssembly(a);
-                    assemblies.Enqueue(a);
+                    LaunchAssemblyLoader(assembly);
                 }
                 catch (Exception ex)
                 {
-                    Debug.WriteLine("Error scanning assembly {0}. {1}", a, ex);
+                    Debug.WriteLine("Error scanning assembly {0}. {1}", assembly, ex);
                 }
             }
+
+            var safeCount = 0;
+            // lets wait until all assemblies are loaded
+            do
+            {
+                if (safeCount++ > 200)
+                {
+                    throw new TimeoutException("Timeout while waiting for assemblies to load");
+                }
+
+                Thread.Sleep(50);
+            } while (pendingAssemblies > 0);
         }
 
 
@@ -76,8 +82,8 @@ internal static void Initialize()
         internal static void Shutdown()
         {
             AppDomain domain = AppDomain.CurrentDomain;
-            domain.AssemblyLoad -= lhandler;
-            domain.AssemblyResolve -= rhandler;
+            domain.AssemblyLoad -= AssemblyLoadHandler;
+            domain.AssemblyResolve -= ResolveHandler;
         }
 
 
@@ -88,13 +94,41 @@ internal static void Shutdown()
         /// so that we can know about assemblies that get loaded after the
         /// Python runtime is initialized.
         /// </summary>
+        /// <remarks>Scanning assemblies here caused internal hangs when calling
+        /// <see cref="Assembly.GetTypes"/></remarks>
         private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args)
         {
             Assembly assembly = args.LoadedAssembly;
-            assemblies.Enqueue(assembly);
-            ScanAssembly(assembly);
+            LaunchAssemblyLoader(assembly);
         }
 
+        /// <summary>
+        /// Launches a new task that will load the provided assembly
+        /// </summary>
+        private static void LaunchAssemblyLoader(Assembly assembly)
+        {
+            if (assembly != null)
+            {
+                Interlocked.Increment(ref pendingAssemblies);
+                Task.Factory.StartNew(() =>
+                {
+                    try
+                    {
+                        if (assembliesNamesCache.TryAdd(assembly.GetName().Name, assembly))
+                        {
+                            assemblies.Enqueue(assembly);
+                            ScanAssembly(assembly);
+                        }
+                    }
+                    catch
+                    {
+                        // pass
+                    }
+
+                    Interlocked.Decrement(ref pendingAssemblies);
+                });
+            }
+        }
 
         /// <summary>
         /// Event handler for assembly resolve events. This is needed because
@@ -106,12 +140,12 @@ private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args)
         private static Assembly ResolveHandler(object ob, ResolveEventArgs args)
         {
             string name = args.Name.ToLower();
-            foreach (Assembly a in assemblies)
+            foreach (var assembly in assemblies)
             {
-                string full = a.FullName.ToLower();
+                var full = assembly.FullName.ToLower();
                 if (full.StartsWith(name))
                 {
-                    return a;
+                    return assembly;
                 }
             }
             return LoadAssemblyPath(args.Name);
@@ -133,54 +167,44 @@ internal static void UpdatePath()
         {
             IntPtr list = Runtime.PySys_GetObject("path");
             int count = Runtime.PyList_Size(list);
+            var sep = Path.DirectorySeparatorChar;
+
             if (count != pypath.Count)
             {
                 pypath.Clear();
                 probed.Clear();
+                // add first the current path
+                pypath.Add("");
                 for (var i = 0; i < count; i++)
                 {
                     IntPtr item = Runtime.PyList_GetItem(list, i);
                     string path = Runtime.GetManagedString(item);
                     if (path != null)
                     {
-                        pypath.Add(path);
+                        pypath.Add(path == string.Empty ? path : path + sep);
                     }
                 }
             }
         }
 
-
         /// <summary>
         /// Given an assembly name, try to find this assembly file using the
         /// PYTHONPATH. If not found, return null to indicate implicit load
         /// using standard load semantics (app base directory then GAC, etc.)
         /// </summary>
         public static string FindAssembly(string name)
         {
-            char sep = Path.DirectorySeparatorChar;
-
-            foreach (string head in pypath)
+            foreach (var head in pypath)
             {
-                string path;
-                if (head == null || head.Length == 0)
-                {
-                    path = name;
-                }
-                else
+                var dll = $"{head}{name}.dll";
+                if (File.Exists(dll))
                 {
-                    path = head + sep + name;
+                    return dll;
                 }
-
-                string temp = path + ".dll";
-                if (File.Exists(temp))
-                {
-                    return temp;
-                }
-
-                temp = path + ".exe";
-                if (File.Exists(temp))
+                var executable = $"{head}{name}.exe";
+                if (File.Exists(executable))
                 {
-                    return temp;
+                    return executable;
                 }
             }
             return null;
@@ -200,10 +224,7 @@ public static Assembly LoadAssembly(string name)
             }
             catch (Exception)
             {
-                //if (!(e is System.IO.FileNotFoundException))
-                //{
-                //    throw;
-                //}
+                // ignored
             }
             return assembly;
         }
@@ -214,7 +235,7 @@ public static Assembly LoadAssembly(string name)
         /// </summary>
         public static Assembly LoadAssemblyPath(string name)
         {
-            string path = FindAssembly(name);
+            var path = FindAssembly(name);
             Assembly assembly = null;
             if (path != null)
             {
@@ -224,6 +245,7 @@ public static Assembly LoadAssemblyPath(string name)
                 }
                 catch (Exception)
                 {
+                    // ignored
                 }
             }
             return assembly;
@@ -251,6 +273,7 @@ public static Assembly LoadAssemblyFullPath(string name)
                     }
                     catch (Exception)
                     {
+                        // ignored
                     }
                 }
             }
@@ -262,14 +285,8 @@ public static Assembly LoadAssemblyFullPath(string name)
         /// </summary>
         public static Assembly FindLoadedAssembly(string name)
         {
-            foreach (Assembly a in assemblies)
-            {
-                if (a.GetName().Name == name)
-                {
-                    return a;
-                }
-            }
-            return null;
+            Assembly result;
+            return assembliesNamesCache.TryGetValue(name, out result) ? result : null;
         }
 
         /// <summary>
@@ -306,10 +323,6 @@ public static bool LoadImplicit(string name, bool warn = true)
                     {
                         a = LoadAssemblyPath(s);
                     }
-                    if (a == null)
-                    {
-                        a = LoadAssembly(s);
-                    }
                     if (a != null && !assembliesSet.Contains(a))
                     {
                         loaded = true;
@@ -344,12 +357,6 @@ internal static void ScanAssembly(Assembly assembly)
             // gather a list of all of the namespaces contributed to by
             // the assembly.
 
-            // skip this assembly, it causes 'GetTypes' call to hang
-            if (assembly.FullName.StartsWith("System.Windows.Forms"))
-            {
-                return;
-            }
-
             Type[] types = assembly.GetTypes();
             foreach (Type t in types)
             {
@@ -361,13 +368,13 @@ internal static void ScanAssembly(Assembly assembly)
                     for (var n = 0; n < names.Length; n++)
                     {
                         s = n == 0 ? names[0] : s + "." + names[n];
-                        namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, string>());
+                        namespaces.TryAdd(s, new ConcurrentDictionary<Assembly, byte>());
                     }
                 }
 
                 if (ns != null)
                 {
-                    namespaces[ns].TryAdd(assembly, string.Empty);
+                    namespaces[ns].TryAdd(assembly, 1);
                 }
 
                 if (ns != null && t.IsGenericTypeDefinition)
@@ -457,11 +464,17 @@ public static List<string> GetNames(string nsname)
         /// </summary>
         public static Type LookupType(string qname)
         {
+            Type type;
+            if (lookupTypeCache.TryGetValue(qname, out type))
+            {
+                return type;
+            }
             foreach (Assembly assembly in assemblies)
             {
-                Type type = assembly.GetType(qname);
+                type = assembly.GetType(qname);
                 if (type != null)
                 {
+                    lookupTypeCache[qname] = type;
                     return type;
                 }
             }

src/runtime/classmanager.cs

@@ -40,9 +40,8 @@ static ClassManager()
         /// </summary>
         internal static ClassBase GetClass(Type type)
         {
-            ClassBase cb = null;
-            cache.TryGetValue(type, out cb);
-            if (cb != null)
+            ClassBase cb;
+            if (cache.TryGetValue(type, out cb))
             {
                 return cb;
             }