recipe: introduce PyProjectRecipe and MesonRecipe

Author: T-Dynamos

Dockerfile

@@ -57,6 +57,7 @@ RUN ${RETRY} apt -y update -qq > /dev/null \
         ant \
         autoconf \
         automake \
+        autopoint \
         ccache \
         cmake \
         g++ \

doc/source/quickstart.rst

@@ -72,6 +72,7 @@ the following command (re-adapted from the `Dockerfile` we use to perform CI bui
         ant \
         autoconf \
         automake \
+        autopoint \
         ccache \
         cmake \
         g++ \

pythonforandroid/recipe.py

@@ -899,6 +899,8 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         # Set the LANG, this isn't usually important but is a better default
         # as it occasionally matters how Python e.g. reads files
         env['LANG'] = "en_GB.UTF-8"
+        # Binaries made by packages installed by pip
+        env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"]
 
         if not self.call_hostpython_via_targetpython:
             env['CFLAGS'] += ' -I{}'.format(
@@ -982,9 +984,17 @@ def install_hostpython_package(self, arch):
     def python_version(self):
         return Recipe.get_recipe("python3", self.ctx).version
 
-    def install_hostpython_prerequisites(self, force_upgrade=True):
-        if len(self.hostpython_prerequisites) == 0:
+    def get_python_formatted_version(self):
+        parsed_version = packaging.version.parse(self.python_version)
+        return f"{parsed_version.major}.{parsed_version.minor}"
+
+    def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
+        if not packages:
+            packages = self.hostpython_prerequisites
+
+        if len(packages) == 0:
             return
+
         pip_options = [
             "install",
             *self.hostpython_prerequisites,
@@ -999,6 +1009,10 @@ def install_hostpython_prerequisites(self, force_upgrade=True):
         # Use system's pip
         shprint(sh.pip, *pip_options)
 
+    def restore_hostpython_prerequisites(self, package):
+        original_version = Recipe.get_recipe(package, self.ctx).version
+        self.install_hostpython_prerequisites(packages=[package + "==" + original_version])
+
 
 class CompiledComponentsPythonRecipe(PythonRecipe):
     pre_build_ext = False
@@ -1243,10 +1257,6 @@ def get_recipe_env(self, arch):
         )
         return env
 
-    def get_python_formatted_version(self):
-        parsed_version = packaging.version.parse(self.python_version)
-        return f"{parsed_version.major}.{parsed_version.minor}"
-
     def check_host_deps(self):
         if not hasattr(sh, "rustup"):
             error(
@@ -1295,6 +1305,145 @@ def build_arch(self, arch):
         info("Successfully installed  '{}'".format(basename(built_wheel)))
 
 
+class PyProjectRecipe(PythonRecipe):
+    '''Recipe for projects which containes `pyproject.toml`'''
+
+    default_deps = ["build", "wheel", "pyproject_hooks",
+                    "setuptools", "packaging", "pyproject_metadata"]
+    # Extra args to pass to `python -m build ...`
+    extra_build_args = []
+    hostpython_prerequisites = default_deps
+    call_hostpython_via_targetpython = False
+
+    def ensure_default_deps(self):
+        for dep in self.default_deps:
+            if dep not in self.hostpython_prerequisites:
+                self.hostpython_prerequisites.append(dep)
+
+    def build_arch(self, arch):
+        self.ensure_default_deps()
+        self.install_hostpython_prerequisites()
+
+        build_dir = self.get_build_dir(arch.arch)
+        env = self.get_recipe_env(arch, with_flags_in_cc=True)
+        built_wheel = None
+        info(str(env))
+
+        # make build dir separatly
+        sub_build_dir = join(build_dir, "p4a_android_build")
+        ensure_dir(sub_build_dir)
+
+        # Copying hostpython to the Python build ensures that it correctly identifies libraries and include files.
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        custom_python = join(python_recipe.get_build_dir(arch), "android-build", "python3")
+        shprint(sh.cp, self.real_hostpython_location, custom_python)
+
+        build_args = [
+            "-m",
+            "build",
+            "--no-isolation",
+            "--skip-dependency-check",
+            "--wheel",
+            "--config-setting",
+            "builddir={}".format(sub_build_dir),
+        ] + self.extra_build_args
+
+        with current_directory(build_dir):
+            shprint(
+                sh.Command(custom_python), *build_args, _env=env
+            )
+            built_wheel = realpath(glob.glob("dist/*.whl")[0])
+
+        info("Unzipping built wheel '{}'".format(basename(built_wheel)))
+
+        with zipfile.ZipFile(built_wheel, "r") as zip_ref:
+            zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch))
+        info("Successfully installed  '{}'".format(basename(built_wheel)))
+
+
+class MesonRecipe(PyProjectRecipe):
+    '''Recipe for projects which uses meson as build system'''
+
+    def sanitize_flags(self, *flag_strings):
+        return " ".join(flag_strings).strip().split(" ")
+
+    def get_recipe_meson_options(self, arch):
+        """Writes python dict to meson config file"""
+        env = self.get_recipe_env(arch, with_flags_in_cc=True)
+        return {
+            "binaries": {
+                "c": arch.get_clang_exe(with_target=True),
+                "cpp": arch.get_clang_exe(with_target=True, plus_plus=True),
+                "ar": self.ctx.ndk.llvm_ar,
+                "strip": self.ctx.ndk.llvm_strip,
+            },
+            "built-in options": {
+                "c_args": self.sanitize_flags(env["CFLAGS"], env["CPPFLAGS"]),
+                "cpp_args": self.sanitize_flags(env["CXXFLAGS"], env["CPPFLAGS"]),
+                "c_link_args": self.sanitize_flags(env["LDFLAGS"]),
+                "cpp_link_args": self.sanitize_flags(env["LDFLAGS"]),
+            },
+            "properties": {
+                "needs_exe_wrapper": True,
+                "sys_root": self.ctx.ndk.sysroot
+            },
+            "host_machine": {
+                "cpu_family": {
+                    "arm64-v8a": "aarch64",
+                    "armeabi-v7a": "arm",
+                    "x86_64": "x86_64",
+                    "x86": "x86"
+                }[arch.arch],
+                "cpu": {
+                    "arm64-v8a": "aarch64",
+                    "armeabi-v7a": "armv7",
+                    "x86_64": "x86_64",
+                    "x86": "i686"
+                }[arch.arch],
+                "endian": "little",
+                "system": "android",
+            }
+        }
+
+    def write_build_options(self, arch):
+        option_data = ""
+        build_options = self.get_recipe_meson_options(arch)
+        for key in build_options.keys():
+            data_chunk = "[{}]".format(key)
+            for subkey in build_options[key].keys():
+                value = build_options[key][subkey]
+                if isinstance(value, int):
+                    value = str(value)
+                elif isinstance(value, str):
+                    value = "'{}'".format(value)
+                elif isinstance(value, bool):
+                    value = "true" if value else "false"
+                elif isinstance(value, list):
+                    value = "['" + "', '".join(value) + "']"
+                data_chunk += "\n" + subkey + " = " + value
+            option_data += data_chunk + "\n\n"
+        return option_data
+
+    def ensure_args(self, *args):
+        for arg in args:
+            if arg not in self.extra_build_args:
+                self.extra_build_args.append(arg)
+
+    def build_arch(self, arch):
+        cross_file = join(self.get_build_dir(arch), "android.meson.cross")
+        info("Writing cross file at: {}".format(cross_file))
+        # write cross config file
+        with open(cross_file, "w") as file:
+            file.write(self.write_build_options(arch))
+            file.close()
+        # set cross file
+        self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file))
+        # ensure ninja
+        if "ninja" not in self.hostpython_prerequisites:
+            self.hostpython_prerequisites.append("ninja")
+        super().build_arch(arch)
+
+
 class TargetPythonRecipe(Recipe):
     '''Class for target python recipes. Sets ctx.python_recipe to point to
     itself, so as to know later what kind of Python was built or used.'''

pythonforandroid/recipes/matplotlib/__init__.py

@@ -7,7 +7,7 @@
 
 class MatplotlibRecipe(CppCompiledComponentsPythonRecipe):
 
-    version = '3.5.2'
+    version = '3.8.4'
     url = 'https://github.com/matplotlib/matplotlib/archive/v{version}.zip'
 
     depends = ['kiwisolver', 'numpy', 'pillow', 'setuptools', 'freetype']

pythonforandroid/recipes/numpy/__init__.py

@@ -1,31 +1,18 @@
-from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
-from pythonforandroid.logger import shprint, info
-from pythonforandroid.util import current_directory
-from multiprocessing import cpu_count
+from pythonforandroid.recipe import Recipe, MesonRecipe
 from os.path import join
-import glob
-import sh
 import shutil
 
 
-class NumpyRecipe(CompiledComponentsPythonRecipe):
+class NumpyRecipe(MesonRecipe):
+    version = 'e59c074842e3f73483afa5ddef031e856b9fd313'  # 2.0.0
+    url = 'git+https://github.com/numpy/numpy'
+    hostpython_prerequisites = ["Cython>=3.0.6", "meson-python>=0.15.0"]
+    extra_build_args = ['-Csetup-args=-Dblas=none', '-Csetup-args=-Dlapack=none']
 
-    version = '1.22.3'
-    url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip'
-    site_packages_name = 'numpy'
-    depends = ["cython"]
-
-    # This build specifically requires setuptools version 59.2.0
-    hostpython_prerequisites = ["setuptools==59.2.0"]
-
-    install_in_hostpython = True
-    call_hostpython_via_targetpython = False
-
-    patches = [
-        join("patches", "remove-default-paths.patch"),
-        join("patches", "add_libm_explicitly_to_build.patch"),
-        join("patches", "ranlib.patch"),
-    ]
+    def get_recipe_meson_options(self, arch):
+        options = super().get_recipe_meson_options(arch)
+        options["properties"]["longdouble_format"] = "IEEE_DOUBLE_LE" if arch.arch in ["armeabi-v7a", "x86"] else "IEEE_QUAD_LE"
+        return options
 
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env = super().get_recipe_env(arch, with_flags_in_cc)
@@ -37,54 +24,19 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         # NPY_DISABLE_SVML=1 allows numpy to build for non-AVX512 CPUs
         # See: https://github.com/numpy/numpy/issues/21196
         env["NPY_DISABLE_SVML"] = "1"
-
+        env["TARGET_PYTHON_EXE"] = join(Recipe.get_recipe(
+                "python3", self.ctx).get_build_dir(arch.arch), "android-build", "python")
         return env
 
     def build_arch(self, arch):
-        self.hostpython_prerequisites = ["setuptools==59.2.0"]
-        self.install_hostpython_prerequisites()
-
         super().build_arch(arch)
-
-        # Post build step to restore setuptools version
-        self.hostpython_prerequisites = ["setuptools=={}".format(
-            Recipe.get_recipe("setuptools", self.ctx).version)
-        ]
-        self.install_hostpython_prerequisites()
-
-    def _build_compiled_components(self, arch):
-        info('Building compiled components in {}'.format(self.name))
-
-        env = self.get_recipe_env(arch)
-        with current_directory(self.get_build_dir(arch.arch)):
-            hostpython = sh.Command(self.hostpython_location)
-            shprint(hostpython, 'setup.py', self.build_cmd, '-v',
-                    _env=env, *self.setup_extra_args)
-            build_dir = glob.glob('build/lib.*')[0]
-            shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
-                    env['STRIP'], '{}', ';', _env=env)
-
-    def _rebuild_compiled_components(self, arch, env):
-        info('Rebuilding compiled components in {}'.format(self.name))
-
-        hostpython = sh.Command(self.real_hostpython_location)
-        shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env)
-        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
-                *self.setup_extra_args)
-
-    def build_compiled_components(self, arch):
-        self.setup_extra_args = ['-j', str(cpu_count())]
-        self._build_compiled_components(arch)
-        self.setup_extra_args = []
-
-    def rebuild_compiled_components(self, arch, env):
-        self.setup_extra_args = ['-j', str(cpu_count())]
-        self._rebuild_compiled_components(arch, env)
-        self.setup_extra_args = []
+        # restore cython version
+        self.restore_hostpython_prerequisites("cython")
 
     def get_hostrecipe_env(self, arch):
         env = super().get_hostrecipe_env(arch)
         env['RANLIB'] = shutil.which('ranlib')
+        env["LDFLAGS"] += " -lm"
         return env