Merge pull request #101 from adafruit/include-binary-files-in-package

Get package name from pyproject.toml, allow arbitrary files in packages

Author: dhalbert

circuitpython_build_tools/build.py

@@ -36,8 +36,48 @@
 import subprocess
 import tempfile
 
+# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
+# and should be removed when the fixed version is incorporated in its respective bundle.
+
+pyproject_py_modules_blacklist = set((
+    # adafruit bundle
+    "adafruit_colorsys",
+
+    # community bundle
+    "at24mac_eeprom",
+    "circuitpython_Candlesticks",
+    "CircuitPython_Color_Picker",
+    "CircuitPython_Equalizer",
+    "CircuitPython_Scales",
+    "circuitPython_Slider",
+    "circuitpython_uboxplot",
+    "P1AM",
+    "p1am_200_helpers",
+))
+
+if sys.version_info >= (3, 11):
+    from tomllib import loads as load_toml
+else:
+    from tomli import loads as load_toml
+
+def load_pyproject_toml(lib_path: pathlib.Path):
+    try:
+        return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        print(f"No pyproject.toml in {lib_path}")
+        return {}
+
+def get_nested(doc, *args, default=None):
+    for a in args:
+        if doc is None: return default
+        try:
+            doc = doc[a]
+        except (KeyError, IndexError) as e:
+            return default
+    return doc
+
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
-GLOB_PATTERNS = ["*.py", "font5x8.bin"]
+GLOB_PATTERNS = ["*.py", "*.bin"]
 S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"
 
 def version_string(path=None, *, valid_semver=False):
@@ -131,17 +171,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
     shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)
 
 def _munge_to_temp(original_path, temp_file, library_version):
-    with open(original_path, "rb") as original_file:
+    with open(original_path, "r", encoding="utf-8") as original_file:
         for line in original_file:
-            if original_path.endswith(".bin"):
-                # this is solely for adafruit_framebuf/examples/font5x8.bin
-                temp_file.write(line)
-            else:
-                line = line.decode("utf-8").strip("\n")
-                if line.startswith("__version__"):
-                    line = line.replace("0.0.0-auto.0", library_version)
-                    line = line.replace("0.0.0+auto.0", library_version)
-                temp_file.write(line.encode("utf-8") + b"\r\n")
+            line = line.strip("\n")
+            if line.startswith("__version__"):
+                line = line.replace("0.0.0-auto.0", library_version)
+                line = line.replace("0.0.0+auto.0", library_version)
+            print(line, file=temp_file)
     temp_file.flush()
 
 def get_package_info(library_path, package_folder_prefix):
@@ -154,61 +190,52 @@ def get_package_info(library_path, package_folder_prefix):
     for pattern in GLOB_PATTERNS:
         glob_search.extend(list(lib_path.rglob(pattern)))
 
-    package_info["is_package"] = False
-    for file in glob_search:
-        if file.parts[parent_idx] != "examples":
-            if len(file.parts) > parent_idx + 1:
-                for prefix in package_folder_prefix:
-                    if file.parts[parent_idx].startswith(prefix):
-                        package_info["is_package"] = True
-            if package_info["is_package"]:
-                package_files.append(file)
-            else:
-                if file.name in IGNORE_PY:
-                    #print("Ignoring:", file.resolve())
-                    continue
-                if file.parent == lib_path:
-                    py_files.append(file)
-
-    if package_files:
-        package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
-    elif py_files:
-        package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
-    else:
-        package_info["module_name"] = None
-
-    try:
-        package_info["version"] = version_string(library_path, valid_semver=True)
-    except ValueError as e:
-        package_info["version"] = version_string(library_path)
-
-    return package_info
-
-def library(library_path, output_directory, package_folder_prefix,
-            mpy_cross=None, example_bundle=False):
-    py_files = []
-    package_files = []
-    example_files = []
-    total_size = 512
-
-    lib_path = pathlib.Path(library_path)
-    parent_idx = len(lib_path.parts)
-    glob_search = []
-    for pattern in GLOB_PATTERNS:
-        glob_search.extend(list(lib_path.rglob(pattern)))
+    pyproject_toml = load_pyproject_toml(lib_path)
+    py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[])
+    packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[])
+
+    blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist]
+
+    if blacklisted:
+        print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml")
+        py_modules = packages = ()
+
+    example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*")
+            if sub_path.is_file()]
+
+    if packages and py_modules:
+        raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")
+
+    elif packages:
+        if len(packages) > 1:
+            raise ValueError("Only a single package is supported")
+        package_name = packages[0]
+        #print(f"Using package name from pyproject.toml: {package_name}")
+        package_info["is_package"] = True
+        package_info["module_name"] = package_name
+        package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*")
+                if sub_path.is_file()]
+
+    elif py_modules:
+        if len(py_modules) > 1:
+            raise ValueError("Only a single module is supported")
+        py_module = py_modules[0]
+        #print(f"Using module name from pyproject.toml: {py_module}")
+        package_name = py_module
+        package_info["is_package"] = False
+        package_info["module_name"] = py_module
+        py_files = [lib_path / f"{py_module}.py"]
 
-    for file in glob_search:
-        if file.parts[parent_idx] == "examples":
-            example_files.append(file)
-        else:
-            if not example_bundle:
-                is_package = False
+    else:
+        print(f"{lib_path}: Using legacy autodetection")
+        package_info["is_package"] = False
+        for file in glob_search:
+            if file.parts[parent_idx] != "examples":
                 if len(file.parts) > parent_idx + 1:
                     for prefix in package_folder_prefix:
                         if file.parts[parent_idx].startswith(prefix):
-                            is_package = True
-
-                if is_package:
+                            package_info["is_package"] = True
+                if package_info["is_package"]:
                     package_files.append(file)
                 else:
                     if file.name in IGNORE_PY:
@@ -217,91 +244,78 @@ def library(library_path, output_directory, package_folder_prefix,
                     if file.parent == lib_path:
                         py_files.append(file)
 
+        if package_files:
+            package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
+        elif py_files:
+            package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
+        else:
+            package_info["module_name"] = None
+
     if len(py_files) > 1:
         raise ValueError("Multiple top level py files not allowed. Please put "
                          "them in a package or combine them into a single file.")
 
-    if package_files:
-        module_name = package_files[0].relative_to(library_path).parent.name
-    elif py_files:
-        module_name = py_files[0].relative_to(library_path).name[:-3]
-    else:
-        module_name = None
+    package_info["package_files"] = package_files
+    package_info["py_files"] = py_files
+    package_info["example_files"] = example_files
+
+    try:
+        package_info["version"] = version_string(library_path, valid_semver=True)
+    except ValueError as e:
+        print(library_path + " has version that doesn't follow SemVer (semver.org)")
+        print(e)
+        package_info["version"] = version_string(library_path)
+
+    return package_info
+
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
+    lib_path = pathlib.Path(library_path)
+    package_info = get_package_info(library_path, package_folder_prefix)
+    py_package_files = package_info["package_files"] + package_info["py_files"]
+    example_files = package_info["example_files"]
+    module_name = package_info["module_name"]
 
     for fn in example_files:
         base_dir = os.path.join(output_directory.replace("/lib", "/"),
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
 
-    for fn in package_files:
+    for fn in py_package_files:
         base_dir = os.path.join(output_directory,
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
 
-    new_extension = ".py"
-    if mpy_cross:
-        new_extension = ".mpy"
+    library_version = package_info['version']
 
-    try:
-        library_version = version_string(library_path, valid_semver=True)
-    except ValueError as e:
-        print(library_path + " has version that doesn't follow SemVer (semver.org)")
-        print(e)
-        library_version = version_string(library_path)
-
-    for filename in py_files:
-        full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(
-            output_directory,
-            filename.relative_to(library_path).with_suffix(new_extension)
-        )
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-            # Windows: close the temp file before it can be read or copied by name
-        if mpy_cross:
-            mpy_success = subprocess.call([
-                mpy_cross,
-                "-o", output_file,
-                "-s", str(filename.relative_to(library_path)),
-                temp_filename
-            ])
-            if mpy_success != 0:
-                raise RuntimeError("mpy-cross failed on", full_path)
-        else:
-            shutil.copyfile(temp_filename, output_file)
-        os.remove(temp_filename)
-
-    for filename in package_files:
-        full_path = os.path.join(library_path, filename)
-        output_file = ""
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-            # Windows: close the temp file before it can be read or copied by name
-        if not mpy_cross or os.stat(full_path).st_size == 0:
-            output_file = os.path.join(output_directory,
-                                       filename.relative_to(library_path))
-            shutil.copyfile(temp_filename, output_file)
-        else:
-            output_file = os.path.join(
-                output_directory,
-                filename.relative_to(library_path).with_suffix(new_extension)
-            )
-
-            mpy_success = subprocess.call([
-                mpy_cross,
-                "-o", output_file,
-                "-s", str(filename.relative_to(library_path)),
-                temp_filename
-            ])
-            if mpy_success != 0:
-                raise RuntimeError("mpy-cross failed on", full_path)
-        os.remove(temp_filename)
+    if not example_bundle:
+        for filename in py_package_files:
+            full_path = os.path.join(library_path, filename)
+            output_file = output_directory / filename.relative_to(library_path)
+            if filename.suffix == ".py":
+                with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
+                    temp_file_name = temp_file.name
+                    try:
+                        _munge_to_temp(full_path, temp_file, library_version)
+                        temp_file.close()
+                        if mpy_cross and os.stat(temp_file.name).st_size != 0:
+                            output_file = output_file.with_suffix(".mpy")
+                            mpy_success = subprocess.call([
+                                mpy_cross,
+                                "-o", output_file,
+                                "-s", str(filename.relative_to(library_path)),
+                                temp_file.name
+                            ])
+                            if mpy_success != 0:
+                                raise RuntimeError("mpy-cross failed on", full_path)
+                        else:
+                            shutil.copyfile(full_path, output_file)
+                    finally:
+                        os.remove(temp_file_name)
+            else:
+                shutil.copyfile(full_path, output_file)
 
     requirements_files = lib_path.glob("requirements.txt*")
     requirements_files = [f for f in requirements_files if f.stat().st_size > 0]
@@ -314,11 +328,9 @@ def library(library_path, output_directory, package_folder_prefix,
         requirements_dir = pathlib.Path(output_directory).parent / "requirements"
         if not os.path.isdir(requirements_dir):
             os.makedirs(requirements_dir, exist_ok=True)
-            total_size += 512
         requirements_subdir = f"{requirements_dir}/{module_name}"
         if not os.path.isdir(requirements_subdir):
             os.makedirs(requirements_subdir, exist_ok=True)
-            total_size += 512
         for filename in requirements_files:
             full_path = os.path.join(library_path, filename)
             output_file = os.path.join(requirements_subdir, filename.name)
@@ -328,9 +340,4 @@ def library(library_path, output_directory, package_folder_prefix,
         full_path = os.path.join(library_path, filename)
         output_file = os.path.join(output_directory.replace("/lib", "/"),
                                    filename.relative_to(library_path))
-        temp_filename = ""
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-        shutil.copyfile(temp_filename, output_file)
-        os.remove(temp_filename)
+        shutil.copyfile(full_path, output_file)

requirements.txt

@@ -2,3 +2,4 @@ Click
 requests
 semver
 wheel
+tomli; python_version < "3.11"