feature: add support for musllinux

Detect the running libc at startup & select the correct policy file based on this.
For musllinux, we select only one musllinux policy depending on the running version of musl.

Author: lkollar

auditwheel/error.py

@@ -0,0 +1,6 @@
+class AuditwheelException(Exception):
+    pass
+
+
+class InvalidLibc(AuditwheelException):
+    pass

auditwheel/lddtree.py

@@ -17,9 +17,12 @@
 import errno
 import logging
 import functools
+from pathlib import Path
 from typing import List, Dict, Optional, Any, Tuple
 
 from elftools.elf.elffile import ELFFile
+from .libc import get_libc, Libc
+
 
 log = logging.getLogger(__name__)
 __all__ = ['lddtree']
@@ -195,11 +198,32 @@ def load_ld_paths(root: str = '/', prefix: str = '') -> Dict[str, List[str]]:
             # on a per-ELF basis so it can get turned into the right thing.
             ldpaths['env'] = parse_ld_paths(env_ldpath, path='')
 
-    # Load up /etc/ld.so.conf.
-    ldpaths['conf'] = parse_ld_so_conf(root + prefix + '/etc/ld.so.conf',
-                                       root=root)
-    # the trusted directories are not necessarily in ld.so.conf
-    ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64'])
+    libc = get_libc()
+    if libc == Libc.MUSL:
+        # from https://git.musl-libc.org/cgit/musl/tree/ldso
+        # /dynlink.c?id=3f701faace7addc75d16dea8a6cd769fa5b3f260#n1063
+        root_prefix = Path(root) / prefix
+        ld_musl = list((root_prefix / 'etc').glob("ld-musl-*.path"))
+        assert len(ld_musl) <= 1
+        if len(ld_musl) == 0:
+            ldpaths['conf'] = [
+                root + '/lib',
+                root + '/usr/local/lib',
+                root + '/usr/lib'
+            ]
+        else:
+            ldpaths['conf'] = []
+            for ldpath in ld_musl[0].read_text().split(':'):
+                ldpath_stripped = ldpath.strip()
+                if ldpath_stripped == "":
+                    continue
+                ldpaths['conf'].append(root + ldpath_stripped)
+    else:
+        # Load up /etc/ld.so.conf.
+        ldpaths['conf'] = parse_ld_so_conf(root + prefix + '/etc/ld.so.conf',
+                                           root=root)
+        # the trusted directories are not necessarily in ld.so.conf
+        ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64'])
     log.debug('linker ldpaths: %s', ldpaths)
     return ldpaths
 

auditwheel/libc.py

@@ -0,0 +1,23 @@
+import logging
+from enum import IntEnum
+
+from .error import InvalidLibc
+from .musllinux import find_musl_libc
+
+
+logger = logging.getLogger(__name__)
+
+
+class Libc(IntEnum):
+    GLIBC = 1,
+    MUSL = 2,
+
+
+def get_libc() -> Libc:
+    try:
+        find_musl_libc()
+        logger.debug("Detected musl libc")
+        return Libc.MUSL
+    except InvalidLibc:
+        logger.debug("Falling back to GNU libc")
+        return Libc.GLIBC

auditwheel/musllinux.py

@@ -0,0 +1,60 @@
+import logging
+import pathlib
+import subprocess
+import re
+from typing import NamedTuple
+
+from auditwheel.error import InvalidLibc
+
+LOG = logging.getLogger(__name__)
+
+
+class MuslVersion(NamedTuple):
+    major: int
+    minor: int
+    patch: int
+
+
+def find_musl_libc() -> pathlib.Path:
+    try:
+        ldd = subprocess.check_output(["ldd", "/bin/ls"], errors='strict')
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        LOG.error("Failed to determine libc version", exc_info=True)
+        raise InvalidLibc
+
+    match = re.search(
+        r"libc\.musl-(?P<platform>\w+)\.so.1 "  # TODO drop the platform
+        r"=> (?P<path>[/\-\w.]+)",
+        ldd)
+
+    if not match:
+        raise InvalidLibc
+
+    return pathlib.Path(match.group("path"))
+
+
+def get_musl_version(ld_path: pathlib.Path) -> MuslVersion:
+    try:
+        ld = subprocess.run(
+            [ld_path],
+            check=False,
+            errors='strict',
+            stderr=subprocess.PIPE
+        ).stderr
+    except FileNotFoundError:
+        LOG.error("Failed to determine musl version", exc_info=True)
+        raise InvalidLibc
+
+    match = re.search(
+        r"Version "
+        r"(?P<major>\d+)."
+        r"(?P<minor>\d+)."
+        r"(?P<patch>\d+)",
+        ld)
+    if not match:
+        raise InvalidLibc
+
+    return MuslVersion(
+        int(match.group("major")),
+        int(match.group("minor")),
+        int(match.group("patch")))

auditwheel/policy/__init__.py

@@ -2,10 +2,16 @@
 import json
 import platform as _platform_module
 from collections import defaultdict
+from pathlib import Path
 from typing import Dict, List, Optional, Set
 from os.path import join, dirname, abspath
 import logging
 
+from ..libc import get_libc, Libc
+from ..musllinux import find_musl_libc, get_musl_version
+
+
+_HERE = Path(__file__).parent
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +28,7 @@ def get_arch_name() -> str:
 
 
 _ARCH_NAME = get_arch_name()
+_LIBC = get_libc()
 
 
 def _validate_pep600_compliance(policies) -> None:
@@ -56,18 +63,64 @@ def _validate_pep600_compliance(policies) -> None:
             symbol_versions[arch] = symbol_versions_arch
 
 
-with open(join(dirname(abspath(__file__)), 'manylinux-policy.json')) as f:
+_POLICY_JSON_MAP = {
+    Libc.GLIBC: _HERE / 'manylinux-policy.json',
+    Libc.MUSL: _HERE / 'musllinux-policy.json',
+}
+
+
+def _get_musl_policy():
+    if _LIBC != Libc.MUSL:
+        return None
+    musl_version = get_musl_version(find_musl_libc())
+    return f'musllinux_{musl_version.major}_{musl_version.minor}'
+
+
+_MUSL_POLICY = _get_musl_policy()
+
+
+def _fixup_musl_libc_soname(whitelist):
+    if _LIBC != Libc.MUSL:
+        return whitelist
+    soname_map = {
+        "libc.so": {
+            "x86_64": "libc.musl-x86_64.so.1",
+            "i686": "libc.musl-x86.so.1",
+            "aarch64": "libc.musl-aarch64.so.1",
+            "s390x": "libc.musl-s390x.so.1",
+            "ppc64le": "libc.musl-ppc64le.so.1",
+            "armv7l": "libc.musl-armv7.so.1",
+        }
+    }
+    new_whitelist = []
+    for soname in whitelist:
+        if soname in soname_map:
+            new_soname = soname_map[soname][_ARCH_NAME]
+            logger.debug(f"Replacing whitelisted '{soname}' by '{new_soname}'")
+            new_whitelist.append(new_soname)
+        else:
+            new_whitelist.append(soname)
+    return new_whitelist
+
+
+with _POLICY_JSON_MAP[_LIBC].open() as f:
     _POLICIES = []
     _policies_temp = json.load(f)
     _validate_pep600_compliance(_policies_temp)
     for _p in _policies_temp:
+        if _MUSL_POLICY is not None and \
+                _p['name'] not in {'linux', _MUSL_POLICY}:
+            continue
         if _ARCH_NAME in _p['symbol_versions'].keys() or _p['name'] == 'linux':
             if _p['name'] != 'linux':
                 _p['symbol_versions'] = _p['symbol_versions'][_ARCH_NAME]
             _p['name'] = _p['name'] + '_' + _ARCH_NAME
             _p['aliases'] = [alias + '_' + _ARCH_NAME
                              for alias in _p['aliases']]
+            _p['lib_whitelist'] = _fixup_musl_libc_soname(_p['lib_whitelist'])
             _POLICIES.append(_p)
+    if _LIBC == Libc.MUSL:
+        assert len(_POLICIES) == 2, _POLICIES
 
 POLICY_PRIORITY_HIGHEST = max(p['priority'] for p in _POLICIES)
 POLICY_PRIORITY_LOWEST = min(p['priority'] for p in _POLICIES)
@@ -78,8 +131,8 @@ def load_policies():
 
 
 def _load_policy_schema():
-    with open(join(dirname(abspath(__file__)), 'policy-schema.json')) as f:
-        schema = json.load(f)
+    with open(join(dirname(abspath(__file__)), 'policy-schema.json')) as f_:
+        schema = json.load(f_)
     return schema
 
 
@@ -124,6 +177,8 @@ def get_replace_platforms(name: str) -> List[str]:
         return []
     if name.startswith('manylinux_'):
         return ['linux_' + '_'.join(name.split('_')[3:])]
+    if name.startswith('musllinux_'):
+        return ['linux_' + '_'.join(name.split('_')[3:])]
     return ['linux_' + '_'.join(name.split('_')[1:])]
 
 

auditwheel/policy/musllinux-policy.json

@@ -0,0 +1,48 @@
+[
+  {"name": "linux",
+   "aliases": [],
+   "priority": 0,
+   "symbol_versions": {},
+   "lib_whitelist": []
+  },
+  {"name": "musllinux_1_1",
+    "aliases": [],
+    "priority": 100,
+    "symbol_versions": {
+      "i686": {
+      },
+      "x86_64": {
+      },
+      "aarch64": {
+      },
+      "ppc64le": {
+      },
+      "s390x": {
+      },
+      "armv7l": {
+      }
+    },
+    "lib_whitelist": [
+      "libc.so"
+    ]},
+    {"name": "musllinux_1_2",
+    "aliases": [],
+    "priority": 90,
+    "symbol_versions": {
+      "i686": {
+      },
+      "x86_64": {
+      },
+      "aarch64": {
+      },
+      "ppc64le": {
+      },
+      "s390x": {
+      },
+      "armv7l": {
+      }
+    },
+    "lib_whitelist": [
+      "libc.so"
+    ]}
+]

tests/unit/test_musllinux.py

@@ -0,0 +1,49 @@
+import subprocess
+from unittest.mock import patch
+
+import pytest
+
+from auditwheel.musllinux import find_musl_libc, get_musl_version
+from auditwheel.error import InvalidLibc
+
+
+@patch("auditwheel.musllinux.subprocess.check_output")
+def test_find_musllinux_no_ldd(check_output_mock):
+    check_output_mock.side_effect = FileNotFoundError()
+    with pytest.raises(InvalidLibc):
+        find_musl_libc()
+
+
+@patch("auditwheel.musllinux.subprocess.check_output")
+def test_find_musllinux_ldd_error(check_output_mock):
+    check_output_mock.side_effect = subprocess.CalledProcessError(1, "ldd")
+    with pytest.raises(InvalidLibc):
+        find_musl_libc()
+
+
+@patch("auditwheel.musllinux.subprocess.check_output")
+def test_find_musllinux_not_found(check_output_mock):
+    check_output_mock.return_value = ""
+    with pytest.raises(InvalidLibc):
+        find_musl_libc()
+
+
+def test_get_musl_version_invalid_path():
+    with pytest.raises(InvalidLibc):
+        get_musl_version("/tmp/no/executable/here")
+
+
+@patch("auditwheel.musllinux.subprocess.run")
+def test_get_musl_version_invalid_version(run_mock):
+    run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 1.1")
+    with pytest.raises(InvalidLibc):
+        get_musl_version("anything")
+
+
+@patch("auditwheel.musllinux.subprocess.run")
+def test_get_musl_version_valid_version(run_mock):
+    run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 5.6.7")
+    version = get_musl_version("anything")
+    assert version.major == 5
+    assert version.minor == 6
+    assert version.patch == 7