Get package name from settings.toml, allow arbitrary files in packages

This is more dependable, and when we know the package name we can
glob inside it to get all files such as bin or ttf files.

This will allow e.g., 5x8.bin & ov5640_autofocus.bin within
bundles.

the behavior of bundlefly and circup when encountering .bin files
needs to be checked.

Tested by building modified pycamera bundle and the autofocus.bin file
appears in the generated zip files:
```
pycamera-py-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096
pycamera-8.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096
pycamera-9.x-mpy-ec67bde/lib/adafruit_pycamera/ov5640_autofocus.bin 4077 4096
```

There's at least one library in the bundle that has incorrect metadata
and that leads to an error:
    adafruit/Adafruit_CircuitPython_Colorsys#29

Author: jepler

circuitpython_build_tools/build.py

@@ -36,8 +36,29 @@
 import subprocess
 import tempfile
 
+if sys.version_info >= (3, 11):
+    from tomllib import loads as load_toml
+else:
+    from tomli import loads as load_toml
+
+def load_settings_toml(lib_path: pathlib.Path):
+    try:
+        return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        print(f"No settings.toml in {lib_path}")
+        return {}
+
+def get_nested(doc, *args, default=None):
+    for a in args:
+        if doc is None: return default
+        try:
+            doc = doc[a]
+        except (KeyError, IndexError) as e:
+            return default
+    return doc
+
 IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
-GLOB_PATTERNS = ["*.py", "font5x8.bin"]
+GLOB_PATTERNS = ["*.py", "*.bin"]
 S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"
 
 def version_string(path=None, *, valid_semver=False):
@@ -131,17 +152,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
     shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)
 
 def _munge_to_temp(original_path, temp_file, library_version):
-    with open(original_path, "rb") as original_file:
+    with open(original_path, "r", encoding="utf-8") as original_file:
         for line in original_file:
-            if original_path.endswith(".bin"):
-                # this is solely for adafruit_framebuf/examples/font5x8.bin
-                temp_file.write(line)
-            else:
-                line = line.decode("utf-8").strip("\n")
-                if line.startswith("__version__"):
-                    line = line.replace("0.0.0-auto.0", library_version)
-                    line = line.replace("0.0.0+auto.0", library_version)
-                temp_file.write(line.encode("utf-8") + b"\r\n")
+            line = line.strip("\n")
+            if line.startswith("__version__"):
+                line = line.replace("0.0.0-auto.0", library_version)
+                line = line.replace("0.0.0+auto.0", library_version)
+            print(line, file=temp_file)
     temp_file.flush()
 
 def get_package_info(library_path, package_folder_prefix):
@@ -154,61 +171,46 @@ def get_package_info(library_path, package_folder_prefix):
     for pattern in GLOB_PATTERNS:
         glob_search.extend(list(lib_path.rglob(pattern)))
 
-    package_info["is_package"] = False
-    for file in glob_search:
-        if file.parts[parent_idx] != "examples":
-            if len(file.parts) > parent_idx + 1:
-                for prefix in package_folder_prefix:
-                    if file.parts[parent_idx].startswith(prefix):
-                        package_info["is_package"] = True
-            if package_info["is_package"]:
-                package_files.append(file)
-            else:
-                if file.name in IGNORE_PY:
-                    #print("Ignoring:", file.resolve())
-                    continue
-                if file.parent == lib_path:
-                    py_files.append(file)
-
-    if package_files:
-        package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
-    elif py_files:
-        package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
-    else:
-        package_info["module_name"] = None
-
-    try:
-        package_info["version"] = version_string(library_path, valid_semver=True)
-    except ValueError as e:
-        package_info["version"] = version_string(library_path)
-
-    return package_info
-
-def library(library_path, output_directory, package_folder_prefix,
-            mpy_cross=None, example_bundle=False):
-    py_files = []
-    package_files = []
-    example_files = []
-    total_size = 512
-
-    lib_path = pathlib.Path(library_path)
-    parent_idx = len(lib_path.parts)
-    glob_search = []
-    for pattern in GLOB_PATTERNS:
-        glob_search.extend(list(lib_path.rglob(pattern)))
-
-    for file in glob_search:
-        if file.parts[parent_idx] == "examples":
-            example_files.append(file)
-        else:
-            if not example_bundle:
-                is_package = False
+    settings_toml = load_settings_toml(lib_path)
+    py_modules = get_nested(settings_toml, "tool", "setuptools", "py-modules", default=[])
+    packages = get_nested(settings_toml, "tool", "setuptools", "packages", default=[])
+
+    example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*")
+            if sub_path.is_file()]
+
+    if packages and py_modules:
+        raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")
+
+    elif packages:
+        if len(packages) > 1:
+            raise ValueError("Only a single package is supported")
+        package_name = packages[0]
+        print(f"Using package name from settings.toml: {package_name}")
+        package_info["is_package"] = True
+        package_info["module_name"] = package_name
+        package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*")
+                if sub_path.is_file()]
+
+    elif py_modules:
+        if len(py_modules) > 1:
+            raise ValueError("Only a single module is supported")
+        print("Using module name from settings.toml")
+        py_module = py_modules[0]
+        package_name = py_module
+        package_info["is_package"] = False
+        package_info["module_name"] = py_module
+        py_files = [lib_path / f"{py_module}.py"]
+
+    if not packages and not py_modules:
+        print("Using legacy autodetection")
+        package_info["is_package"] = False
+        for file in glob_search:
+            if file.parts[parent_idx] != "examples":
                 if len(file.parts) > parent_idx + 1:
                     for prefix in package_folder_prefix:
                         if file.parts[parent_idx].startswith(prefix):
-                            is_package = True
-
-                if is_package:
+                            package_info["is_package"] = True
+                if package_info["is_package"]:
                     package_files.append(file)
                 else:
                     if file.name in IGNORE_PY:
@@ -217,91 +219,78 @@ def library(library_path, output_directory, package_folder_prefix,
                     if file.parent == lib_path:
                         py_files.append(file)
 
+        if package_files:
+            package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
+        elif py_files:
+            package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
+        else:
+            package_info["module_name"] = None
+
     if len(py_files) > 1:
         raise ValueError("Multiple top level py files not allowed. Please put "
                          "them in a package or combine them into a single file.")
 
-    if package_files:
-        module_name = package_files[0].relative_to(library_path).parent.name
-    elif py_files:
-        module_name = py_files[0].relative_to(library_path).name[:-3]
-    else:
-        module_name = None
+    package_info["package_files"] = package_files
+    package_info["py_files"] = py_files
+    package_info["example_files"] = example_files
+
+    try:
+        package_info["version"] = version_string(library_path, valid_semver=True)
+    except ValueError as e:
+        print(library_path + " has version that doesn't follow SemVer (semver.org)")
+        print(e)
+        package_info["version"] = version_string(library_path)
+
+    return package_info
+
+def library(library_path, output_directory, package_folder_prefix,
+            mpy_cross=None, example_bundle=False):
+    lib_path = pathlib.Path(library_path)
+    package_info = get_package_info(library_path, package_folder_prefix)
+    py_package_files = package_info["package_files"] + package_info["py_files"]
+    example_files = package_info["example_files"]
+    module_name = package_info["module_name"]
 
     for fn in example_files:
         base_dir = os.path.join(output_directory.replace("/lib", "/"),
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
 
-    for fn in package_files:
+    for fn in py_package_files:
         base_dir = os.path.join(output_directory,
                                 fn.relative_to(library_path).parent)
         if not os.path.isdir(base_dir):
             os.makedirs(base_dir)
-            total_size += 512
-
-    new_extension = ".py"
-    if mpy_cross:
-        new_extension = ".mpy"
-
-    try:
-        library_version = version_string(library_path, valid_semver=True)
-    except ValueError as e:
-        print(library_path + " has version that doesn't follow SemVer (semver.org)")
-        print(e)
-        library_version = version_string(library_path)
 
-    for filename in py_files:
-        full_path = os.path.join(library_path, filename)
-        output_file = os.path.join(
-            output_directory,
-            filename.relative_to(library_path).with_suffix(new_extension)
-        )
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-            # Windows: close the temp file before it can be read or copied by name
-        if mpy_cross:
-            mpy_success = subprocess.call([
-                mpy_cross,
-                "-o", output_file,
-                "-s", str(filename.relative_to(library_path)),
-                temp_filename
-            ])
-            if mpy_success != 0:
-                raise RuntimeError("mpy-cross failed on", full_path)
-        else:
-            shutil.copyfile(temp_filename, output_file)
-        os.remove(temp_filename)
+    library_version = package_info['version']
 
-    for filename in package_files:
-        full_path = os.path.join(library_path, filename)
-        output_file = ""
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-            # Windows: close the temp file before it can be read or copied by name
-        if not mpy_cross or os.stat(full_path).st_size == 0:
-            output_file = os.path.join(output_directory,
-                                       filename.relative_to(library_path))
-            shutil.copyfile(temp_filename, output_file)
-        else:
-            output_file = os.path.join(
-                output_directory,
-                filename.relative_to(library_path).with_suffix(new_extension)
-            )
-
-            mpy_success = subprocess.call([
-                mpy_cross,
-                "-o", output_file,
-                "-s", str(filename.relative_to(library_path)),
-                temp_filename
-            ])
-            if mpy_success != 0:
-                raise RuntimeError("mpy-cross failed on", full_path)
-        os.remove(temp_filename)
+    if not example_bundle:
+        for filename in py_package_files:
+            full_path = os.path.join(library_path, filename)
+            output_file = output_directory / filename.relative_to(library_path)
+            if filename.suffix == ".py":
+                with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
+                    temp_file_name = temp_file.name
+                    try:
+                        _munge_to_temp(full_path, temp_file, library_version)
+                        temp_file.close()
+                        if mpy_cross:
+                            output_file = output_file.with_suffix(".mpy")
+                            mpy_success = subprocess.call([
+                                mpy_cross,
+                                "-o", output_file,
+                                "-s", str(filename.relative_to(library_path)),
+                                temp_file.name
+                            ])
+                            if mpy_success != 0:
+                                raise RuntimeError("mpy-cross failed on", full_path)
+                        else:
+                            shutil.copyfile(full_path, output_file)
+                    finally:
+                        os.remove(temp_file_name)
+            else:
+                shutil.copyfile(full_path, output_file)
 
     requirements_files = lib_path.glob("requirements.txt*")
     requirements_files = [f for f in requirements_files if f.stat().st_size > 0]
@@ -314,11 +303,9 @@ def library(library_path, output_directory, package_folder_prefix,
         requirements_dir = pathlib.Path(output_directory).parent / "requirements"
         if not os.path.isdir(requirements_dir):
             os.makedirs(requirements_dir, exist_ok=True)
-            total_size += 512
         requirements_subdir = f"{requirements_dir}/{module_name}"
         if not os.path.isdir(requirements_subdir):
             os.makedirs(requirements_subdir, exist_ok=True)
-            total_size += 512
         for filename in requirements_files:
             full_path = os.path.join(library_path, filename)
             output_file = os.path.join(requirements_subdir, filename.name)
@@ -328,9 +315,4 @@ def library(library_path, output_directory, package_folder_prefix,
         full_path = os.path.join(library_path, filename)
         output_file = os.path.join(output_directory.replace("/lib", "/"),
                                    filename.relative_to(library_path))
-        temp_filename = ""
-        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            _munge_to_temp(full_path, temp_file, library_version)
-            temp_filename = temp_file.name
-        shutil.copyfile(temp_filename, output_file)
-        os.remove(temp_filename)
+        shutil.copyfile(full_path, output_file)

requirements.txt

@@ -2,3 +2,4 @@ Click
 requests
 semver
 wheel
+tomli; python_version < "3.11"