Merge pull request pythonnet#1365 from losttech/features/VersionIndependent

Build single Python.Runtime.dll for all platforms

Author: lostmsu

.github/workflows/main.yml

@@ -12,9 +12,33 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows, ubuntu, macos]
-        python: [3.6, 3.7, 3.8, 3.9]
+        pyver_minor: [6, 7, 8, 9]
         platform: [x64]
         shutdown_mode: [Normal, Soft]
+        include:
+        - os: ubuntu
+          pyver_minor: 6
+          dll_suffix: m
+        - os: ubuntu
+          pyver_minor: 7
+          dll_suffix: m
+
+        - os: macos
+          dll_prefix: lib
+          dll_pyver_major: '3.'
+          dll_suffix: m
+        - os: ubuntu
+          dll_prefix: lib
+          dll_pyver_major: '3.'
+        - os: windows
+          dll_pyver_major: '3'
+
+        - os: ubuntu
+          dll_ext: .so
+        - os: windows
+          dll_ext: .dll
+        - os: macos
+          dll_ext: .dylib
 
     env:
       PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }}
@@ -32,10 +56,10 @@ jobs:
       - name: Setup .NET
         uses: actions/setup-dotnet@v1
 
-      - name: Set up Python ${{ matrix.python }}
+      - name: Set up Python 3.${{ matrix.pyver_minor }}
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python }}
+          python-version: 3.${{ matrix.pyver_minor }}
           architecture: ${{ matrix.platform }}
 
       - name: Install dependencies
@@ -47,16 +71,28 @@ jobs:
           python setup.py configure
           pip install -v .
 
+      # TODO this should be gone once clr module sets PythonDLL or preloads it
+      - name: Python Tests
+        run: pytest
+        if: ${{ matrix.os != 'macos' }}
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
+
       - name: Python Tests
         run: pytest
+        if: ${{ matrix.os == 'macos' }}
 
       - name: Embedding tests
         run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/
         if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
 
       - name: Python tests run from .NET
         run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/
         if: ${{ matrix.os == 'windows' }} # Not working for others right now
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
 
       # TODO: Run perf tests
       # TODO: Run mono tests on Windows?

CHANGELOG.md

@@ -34,6 +34,8 @@ details about the cause of the failure
 when .NET expects an integer [#1342][i1342]
 -   More specific error messages for method argument mismatch
 -   BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters.
+-   BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name
+or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions.
 
 ### Fixed
 
@@ -57,6 +59,8 @@ when .NET expects an integer [#1342][i1342]
 ### Removed
 
 -   implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
+-   support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0
+(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support))
 
 ## [2.5.0][] - 2020-06-14
 

Directory.Build.props

@@ -8,7 +8,11 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="NonCopyableAnalyzer" Version="0.6.0">
+    <PackageReference Include="Microsoft.Net.Compilers.Toolset" Version="3.9.0-3.final">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Lost.NonCopyableAnalyzer" Version="0.7.0-m04">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

appveyor.yml

@@ -30,6 +30,7 @@ environment:
 init:
   # Update Environment Variables based on matrix/platform
   - set PY_VER=%PYTHON_VERSION:.=%
+  - set PYTHONNET_PYDLL=python%PY_VER%.dll
   - set PYTHON=C:\PYTHON%PY_VER%
   - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64)
 

src/console/pythonconsole.cs

@@ -26,7 +26,7 @@ private PythonConsole()
         [STAThread]
         public static int Main(string[] args)
         {
-            // Only net40 is capable to safely inject python.runtime.dll into resources.
+            // Only .NET Framework is capable to safely inject python.runtime.dll into resources.
 #if NET40
             // reference the static assemblyLoader to stop it being optimized away
             AssemblyLoader a = assemblyLoader;

src/embed_tests/References.cs

@@ -23,7 +23,7 @@ public void Dispose()
         public void MoveToPyObject_SetsNull()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 Assert.IsFalse(reference.IsNull());
@@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull()
         public void CanBorrowFromNewReference()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference));

src/embed_tests/TestDomainReload.cs

@@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName)
             // assembly (and Python .NET) to reside
             var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
 
-            theProxy.Call("InitPython", ShutdownMode.Soft);
+            theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
             // From now on use the Proxy to call into the new assembly
             theProxy.RunPython();
 
@@ -400,7 +400,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
                     arg = caller.Execute(arg);
@@ -418,7 +418,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
                     caller.Execute(arg);
@@ -478,8 +478,9 @@ public static void RunPython()
 
         private static IntPtr _state;
 
-        public static void InitPython(ShutdownMode mode)
+        public static void InitPython(ShutdownMode mode, string dllName)
         {
+            PyRuntime.PythonDLL = dllName;
             PythonEngine.Initialize(mode: mode);
             _state = PythonEngine.BeginAllowThreads();
         }

src/embed_tests/TestRuntime.cs

@@ -96,13 +96,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
             // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check.
             var threading = Runtime.Runtime.PyImport_ImportModule("threading");
             Exceptions.ErrorCheck(threading);
-            var threadingDict = Runtime.Runtime.PyModule_GetDict(threading);
+            var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading));
             Exceptions.ErrorCheck(threadingDict);
             var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
-            if (lockType == IntPtr.Zero)
+            if (lockType.IsNull)
                 throw new KeyNotFoundException("class 'Lock' was not found in 'threading'");
 
-            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0));
+            var args = Runtime.Runtime.PyTuple_New(0);
+            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args);
+            Runtime.Runtime.XDecref(args);
             Exceptions.ErrorCheck(lockInstance);
 
             Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance));

src/embed_tests/TestTypeManager.cs

@@ -10,10 +10,11 @@ readonly ref struct BorrowedReference
         readonly IntPtr pointer;
         public bool IsNull => this.pointer == IntPtr.Zero;
 
-
         /// <summary>Gets a raw pointer to the Python object</summary>
         public IntPtr DangerousGetAddress()
             => this.IsNull ? throw new NullReferenceException() : this.pointer;
+        /// <summary>Gets a raw pointer to the Python object</summary>
+        public IntPtr DangerousGetAddressOrNull() => this.pointer;
 
         /// <summary>
         /// Creates new instance of <see cref="BorrowedReference"/> from raw pointer. Unsafe.