diff --git a/signingscript/Dockerfile b/signingscript/Dockerfile index 58ce579f5..e53ff7ebf 100644 --- a/signingscript/Dockerfile +++ b/signingscript/Dockerfile @@ -11,6 +11,7 @@ RUN groupadd --gid 10001 app && \ # Copy only required folders COPY ["signingscript", "/app/signingscript/"] +COPY ["scriptworker_client", "/app/scriptworker_client/"] COPY ["configloader", "/app/configloader/"] COPY ["docker.d", "/app/docker.d/"] COPY ["vendored", "/app/vendored/"] @@ -19,19 +20,14 @@ COPY ["vendored", "/app/vendored/"] COPY ["version.jso[n]", "/app/"] # Change owner of /app to app:app +# Build and install libdmg_hfsplus # Install msix # Install rcodesign -RUN chown -R app:app /app && \ - cd /app/signingscript/docker.d && \ - bash build_msix_packaging.sh && \ - cp msix-packaging/.vs/bin/makemsix /usr/bin && \ - cp msix-packaging/.vs/lib/libmsix.so /usr/lib && \ - cd .. && \ - rm -rf msix-packaging && \ - wget -qO- \ - https://github.com/indygreg/apple-platform-rs/releases/download/apple-codesign%2F0.22.0/apple-codesign-0.22.0-x86_64-unknown-linux-musl.tar.gz \ - | tar xvz -C /usr/bin --transform 's/.*\///g' --wildcards --no-anchored 'rcodesign' && \ - chmod +x /usr/bin/rcodesign +RUN chown -R app:app /app \ + && cd /app/signingscript/docker.d \ + && bash build_libdmg_hfsplus.sh /usr/bin \ + && bash install_rcodesign.sh /usr/bin \ + && bash build_msix_packaging.sh # Set user and workdir USER app @@ -39,6 +35,7 @@ WORKDIR /app # Install signingscript + configloader + widevine RUN python -m venv /app \ + && /app/bin/pip install /app/scriptworker_client \ && cd signingscript \ && /app/bin/pip install -r requirements/base.txt \ && /app/bin/pip install . \ diff --git a/signingscript/docker.d/apple_signing_creds.yml b/signingscript/docker.d/apple_signing_creds.yml new file mode 100644 index 000000000..64432a5df --- /dev/null +++ b/signingscript/docker.d/apple_signing_creds.yml @@ -0,0 +1,24 @@ +$let: + scope_prefix: + $match: + 'COT_PRODUCT == "firefox"': 'project:releng:signing:' + 'COT_PRODUCT == "thunderbird"': 'project:comm:thunderbird:releng:signing:' + 'COT_PRODUCT == "mozillavpn"': 'project:mozillavpn:releng:signing:' + 'COT_PRODUCT == "adhoc"': 'project:adhoc:releng:signing:' +in: + $merge: + $match: + 'ENV == "prod" && scope_prefix': + '${scope_prefix[0]}cert:release-signing': + - "app_pkcs12_bundle": {"$eval": "APPLE_APP_SIGNING_PKCS12"} + "installer_pkcs12_bundle": {"$eval": "APPLE_INSTALLER_SIGNING_PKCS12"} + "pkcs12_password": {"$eval": "APPLE_SIGNING_PKCS12_PASSWORD"} + '${scope_prefix[0]}cert:nightly-signing': + - "app_pkcs12_bundle": {"$eval": "APPLE_APP_SIGNING_PKCS12"} + "installer_pkcs12_bundle": {"$eval": "APPLE_INSTALLER_SIGNING_PKCS12"} + "pkcs12_password": {"$eval": "APPLE_SIGNING_PKCS12_PASSWORD"} + 'ENV != "prod" && scope_prefix': + '${scope_prefix[0]}cert:dep-signing': + - "app_pkcs12_bundle": {"$eval": "APPLE_APP_SIGNING_DEP_PKCS12"} + "installer_pkcs12_bundle": {"$eval": "APPLE_INSTALLER_SIGNING_DEP_PKCS12"} + "pkcs12_password": {"$eval": "APPLE_SIGNING_DEP_PKCS12_PASSWORD"} diff --git a/signingscript/docker.d/build_libdmg_hfsplus.sh b/signingscript/docker.d/build_libdmg_hfsplus.sh new file mode 100755 index 000000000..338d94b27 --- /dev/null +++ b/signingscript/docker.d/build_libdmg_hfsplus.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -x -e -v + +# This script is for building libdmg-hfsplus to get the `dmg` and `hfsplus` +# tools for handling DMG archives on Linux. + +DEST=$1 +if [ -d "$DEST" ]; then + echo "Binaries will be installed to: $DEST" +else + echo "Destination directory doesn't exist!" + exit 1 +fi + +git clone --depth=1 --branch mozilla --single-branch https://github.com/mozilla/libdmg-hfsplus/ libdmg-hfsplus + +pushd libdmg-hfsplus + +# The openssl libraries in the sysroot cannot be linked in a PIE executable so we use -no-pie +cmake \ + -DOPENSSL_USE_STATIC_LIBS=1 \ + -DCMAKE_EXE_LINKER_FLAGS=-no-pie \ + . + +make VERBOSE=1 -j$(nproc) + +# We only need the dmg and hfsplus tools. +strip dmg/dmg hfs/hfsplus +cp dmg/dmg hfs/hfsplus "$DEST" + +popd +rm -rf libdmg-hfsplus +echo "Done." diff --git a/signingscript/docker.d/build_msix_packaging.sh b/signingscript/docker.d/build_msix_packaging.sh index 1aa5cd8cc..3fecbb64f 100755 --- a/signingscript/docker.d/build_msix_packaging.sh +++ b/signingscript/docker.d/build_msix_packaging.sh @@ -8,3 +8,8 @@ cd msix-packaging ./makelinux.sh --pack cd .. + +cp msix-packaging/.vs/bin/makemsix /usr/bin +cp msix-packaging/.vs/lib/libmsix.so /usr/lib + +rm -rf msix-packaging diff --git a/signingscript/docker.d/init_worker.sh b/signingscript/docker.d/init_worker.sh index bb1b5940a..0bf63c209 100755 --- a/signingscript/docker.d/init_worker.sh +++ b/signingscript/docker.d/init_worker.sh @@ -21,11 +21,12 @@ test_var_set 'PROJECT_NAME' test_var_set 'PUBLIC_IP' test_var_set 'TEMPLATE_DIR' -export DMG_PATH=$APP_DIR/signingscript/files/dmg -export HFSPLUS_PATH=$APP_DIR/signingscript/files/hfsplus +export DMG_PATH=/usr/bin/dmg +export HFSPLUS_PATH=/usr/bin/hfsplus export PASSWORDS_PATH=$CONFIG_DIR/passwords.json export APPLE_NOTARIZATION_CREDS_PATH=$CONFIG_DIR/apple_notarization_creds.json +export APPLE_SIGNING_CONFIG_PATH=$CONFIG_DIR/apple_signing_config.json export GPG_PUBKEY_PATH=$APP_DIR/signingscript/src/signingscript/data/gpg_pubkey_dep.asc export WIDEVINE_CERT_PATH=$CONFIG_DIR/widevine.crt export AUTHENTICODE_TIMESTAMP_STYLE=old @@ -260,3 +261,4 @@ esac $CONFIG_LOADER $TEMPLATE_DIR/passwords.yml $PASSWORDS_PATH $CONFIG_LOADER $TEMPLATE_DIR/apple_notarization_creds.yml $APPLE_NOTARIZATION_CREDS_PATH +$CONFIG_LOADER $TEMPLATE_DIR/apple_signing_creds.yml $APPLE_SIGNING_CONFIG_PATH diff --git a/signingscript/docker.d/install_rcodesign.sh b/signingscript/docker.d/install_rcodesign.sh new file mode 100755 index 000000000..f63a78ef9 --- /dev/null +++ b/signingscript/docker.d/install_rcodesign.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x -e -v + +DEST=$1 +if [ -d "$DEST" ]; then + echo "Binaries will be installed to: $DEST" +else + echo "Destination directory doesn't exist!" + exit 1 +fi + + +wget -qO- https://github.com/indygreg/apple-platform-rs/releases/download/apple-codesign%2F0.26.0/apple-codesign-0.26.0-x86_64-unknown-linux-musl.tar.gz \ + | tar xvz -C "$DEST" --transform 's/.*\///g' --wildcards --no-anchored 'rcodesign' + +chmod +x "${DEST}/rcodesign" diff --git a/signingscript/docker.d/worker.yml b/signingscript/docker.d/worker.yml index 989f734af..80d8ea4d8 100644 --- a/signingscript/docker.d/worker.yml +++ b/signingscript/docker.d/worker.yml @@ -4,6 +4,7 @@ verbose: { "$eval": "VERBOSE == 'true'" } my_ip: { "$eval": "PUBLIC_IP" } autograph_configs: { "$eval": "PASSWORDS_PATH" } apple_notarization_configs: { "$eval": "APPLE_NOTARIZATION_CREDS_PATH" } +apple_signing_configs: { "$eval": "APPLE_SIGNING_CONFIG_PATH" } taskcluster_scope_prefixes: $flatten: $match: diff --git a/signingscript/files/README b/signingscript/files/README deleted file mode 100644 index b02a745df..000000000 --- a/signingscript/files/README +++ /dev/null @@ -1,3 +0,0 @@ -Built from https://github.com/andreas56/libdmg-hfsplus rev 81dd75fd1549b24bf8af9736ac25518b367e6b63. -Source is available in tooltool, digest bdd80489477647559a99bae8ee9b87d7cc5df26e44bc5f9282b3b1d7eb0e6946b4c61b6159d74fe2422196c9a0c5117f4e8ff4decf7e514f86df8777c0c6dc65 /home/worker/workspace/artifacts/libdmg-hfsplus.tar.xz. - diff --git a/signingscript/files/dmg b/signingscript/files/dmg deleted file mode 100755 index cbd0f9e9f..000000000 Binary files a/signingscript/files/dmg and /dev/null differ diff --git a/signingscript/files/hfsplus b/signingscript/files/hfsplus deleted file mode 100755 index 80fa69171..000000000 Binary files a/signingscript/files/hfsplus and /dev/null differ diff --git a/signingscript/setup.py b/signingscript/setup.py index 61f0b9c33..237a32e00 100644 --- a/signingscript/setup.py +++ b/signingscript/setup.py @@ -5,7 +5,8 @@ with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "version.txt")) as f: version = f.read().rstrip() -install_requires = ["arrow", "mar", "scriptworker", "taskcluster", "mohawk", "winsign", "macholib"] +with open(os.path.join(os.path.abspath(os.path.dirname(__file__)), "requirements", "base.in")) as f: + install_requires = ["scriptworker_client"] + f.readlines() setup( name="signingscript", diff --git a/signingscript/src/signingscript/apple.py b/signingscript/src/signingscript/apple.py new file mode 100644 index 000000000..4c86f0c61 --- /dev/null +++ b/signingscript/src/signingscript/apple.py @@ -0,0 +1,42 @@ +import logging +import os +from shutil import copy2 + +from signingscript.exceptions import SigningScriptError + +log = logging.getLogger(__name__) + + +PROVISIONING_PROFILE_FILENAMES = { + "firefox": "orgmozillafirefox.provisionprofile", + "devedition": "orgmozillafirefoxdeveloperedition.provisionprofile", + "nightly": "orgmozillanightly.provisionprofile", +} + + +def copy_provisioning_profiles(bundlepath, configs): + """Copy provisioning profiles inside bundle + Args: + bundlepath (str): The absolute path to the app bundle + configs (list): The list of configs with schema [{"profile_name": str, "target_path": str}] + """ + for cfg in configs: + profile_name = cfg.get("profile_name") + target_path = cfg.get("target_path") + if not profile_name or not target_path: + raise SigningScriptError(f"profile_name and target_path are required. Got: {cfg}") + + if profile_name not in PROVISIONING_PROFILE_FILENAMES.values(): + raise SigningScriptError(f"profile_name not allowed: {profile_name}") + + profile_path = os.path.join(os.path.dirname(__file__), "data", profile_name) + if not os.path.exists(profile_path): + raise SigningScriptError(f"Provisioning profile not found: {profile_name}") + + # Resolve absolute destination path + target_abs_path = os.path.join(bundlepath, target_path if target_path[0] != "/" else target_path[1:]) + if os.path.exists(target_abs_path): + log.warning("Provisioning profile at {target_path} already exists, overriding.") + + log.info(f"Copying {profile_name} to {target_abs_path}") + copy2(profile_path, target_abs_path) diff --git a/signingscript/src/signingscript/data/orgmozillafirefox.provisionprofile b/signingscript/src/signingscript/data/orgmozillafirefox.provisionprofile new file mode 100644 index 000000000..e1c4920fc Binary files /dev/null and b/signingscript/src/signingscript/data/orgmozillafirefox.provisionprofile differ diff --git a/signingscript/src/signingscript/data/orgmozillafirefoxdeveloperedition.provisionprofile b/signingscript/src/signingscript/data/orgmozillafirefoxdeveloperedition.provisionprofile new file mode 100644 index 000000000..26da37f77 Binary files /dev/null and b/signingscript/src/signingscript/data/orgmozillafirefoxdeveloperedition.provisionprofile differ diff --git a/signingscript/src/signingscript/data/orgmozillanightly.provisionprofile b/signingscript/src/signingscript/data/orgmozillanightly.provisionprofile new file mode 100644 index 000000000..32e0606ee Binary files /dev/null and b/signingscript/src/signingscript/data/orgmozillanightly.provisionprofile differ diff --git a/signingscript/src/signingscript/rcodesign.py b/signingscript/src/signingscript/rcodesign.py index 479fb7cd8..60b0d1586 100644 --- a/signingscript/src/signingscript/rcodesign.py +++ b/signingscript/src/signingscript/rcodesign.py @@ -1,8 +1,14 @@ #!/usr/bin/env python """Functions that interface with rcodesign""" import asyncio +from collections import namedtuple import logging +import os import re +from glob import glob + +from scriptworker_client.aio import download_file, raise_future_exceptions, retry_async +from scriptworker_client.exceptions import DownloadError from signingscript.exceptions import SigningScriptError log = logging.getLogger(__name__) @@ -41,13 +47,14 @@ async def _execute_command(command): stderr = (await proc.stderr.readline()).decode("utf-8").rstrip() if stderr: # Unfortunately a lot of outputs from rcodesign come out to stderr - log.warn(stderr) + log.warning(stderr) output_lines.append(stderr) exitcode = await proc.wait() log.info("exitcode {}".format(exitcode)) return exitcode, output_lines + def find_submission_id(logs): """Given notarization logs, find and return the submission id Args: @@ -128,6 +135,7 @@ async def rcodesign_check_result(logs): raise RCodesignError("Notarization failed!") return + async def rcodesign_staple(path): """Staples a given app Args: @@ -146,3 +154,134 @@ async def rcodesign_staple(path): if exitcode > 0: raise RCodesignError(f"Error stapling notarization. Exit code {exitcode}") return + + +def _create_empty_entitlements_file(dest): + contents = """ + + + + + + """.lstrip() + with open(dest, "wt") as fd: + fd.writelines(contents) + + +async def _download_entitlements(hardened_sign_config, workdir): + """Download entitlements listed in the hardened signing config + Args: + hardened_sign_config (list): hardened signing configs + workdir (str): current work directory where entitlements will be saved + + Returns: + Map of url -> local file location + """ + empty_file = os.path.join(workdir, "0-empty.xml") + _create_empty_entitlements_file(empty_file) + # rcodesign requires us to specify an "empty" entitlements file + url_map = {None: empty_file} + + # Unique urls to be downloaded + urls_to_download = set([i["entitlements"] for i in hardened_sign_config if "entitlements" in i]) + # If nothing found, skip + if not urls_to_download: + log.warn("No entitlements urls provided! Skipping download.") + return url_map + + futures = [] + for index, url in enumerate(urls_to_download, start=1): + # Prefix filename with an index in case filenames are the same + filename = "{}-{}".format(index, url.split("/")[-1]) + dest = os.path.join(workdir, filename) + url_map[url] = dest + log.info(f"Downloading resource: {filename} from {url}") + futures.append( + asyncio.ensure_future( + retry_async( + download_file, + retry_exceptions=(DownloadError, TimeoutError), + args=(url, dest), + attempts=5, + ) + ) + ) + await raise_future_exceptions(futures) + return url_map + + +EntitlementEntry = namedtuple( + "EntitlementEntry", + ["file", "entitlement", "runtime"], +) + +def _get_entitlements_args(hardened_sign_config, path, entitlements_map): + """Builds the list of entitlements based on files in path + + Args: + hardened_sign_config (list): hardened signing configuration + path (str): path to app + """ + entries = [] + + for config in hardened_sign_config: + entitlement_path = entitlements_map.get(config.get("entitlements")) + for path_glob in config["globs"]: + separator = "" + if not path_glob.startswith("/"): + separator = "/" + # Join incoming glob with root of app path + full_path_glob = path + separator + path_glob + for binary_path in glob(full_path_glob, recursive=True): + # Get relative path + relative_path = os.path.relpath(binary_path, path) + # Append ":" to list of args + entries.append( + EntitlementEntry( + file=relative_path, + entitlement=entitlement_path, + runtime=config.get("runtime"), + ) + ) + + return entries + + +async def rcodesign_sign(workdir, path, creds_path, creds_pass_path, hardened_sign_config=[]): + """Signs a given app + Args: + workdir (str): Path to work directory + path (str): Path to be signed + creds_path (str): Path to credentials file + creds_pass_path (str): Path to credentials password file + hardened_sign_config (list): Hardened signing configuration + + Returns: + (Tuple) exit code, log lines + """ + # TODO: Validate and sanitize input + command = [ + "rcodesign", + "sign", + "--code-signature-flags=runtime", + f"--p12-file={creds_path}", + f"--p12-password-file={creds_pass_path}", + ] + + entitlements_map = await _download_entitlements(hardened_sign_config, workdir) + file_entitlements = _get_entitlements_args(hardened_sign_config, path, entitlements_map) + + def _scoped_arg(arg, basepath, value): + if basepath == ".": + return f"--{arg}={value}" + return f"--{arg}={basepath}:{value}" + + for entry in file_entitlements: + if entry.runtime: + flags_arg = _scoped_arg("code-signature-flags", entry.file, "runtime") + command.append(flags_arg) + entitlement_arg = _scoped_arg("entitlements-xml-path", entry.file, entry.entitlement) + command.append(entitlement_arg) + + command.append(path) + await _execute_command(command) diff --git a/signingscript/src/signingscript/script.py b/signingscript/src/signingscript/script.py index 99b8f6a6b..a60a3991d 100755 --- a/signingscript/src/signingscript/script.py +++ b/signingscript/src/signingscript/script.py @@ -1,16 +1,17 @@ #!/usr/bin/env python """Signing script.""" +import base64 +import json import logging import os +from dataclasses import asdict import aiohttp -import json import scriptworker.client -from dataclasses import asdict -from signingscript.task import build_filelist_dict, sign, task_signing_formats, task_cert_type -from signingscript.utils import copy_to_dir, load_apple_notarization_configs, load_autograph_configs from signingscript.exceptions import SigningScriptError +from signingscript.task import build_filelist_dict, sign, task_cert_type, task_signing_formats +from signingscript.utils import copy_to_dir, load_apple_notarization_configs, load_apple_signing_configs, load_autograph_configs, unlink log = logging.getLogger(__name__) @@ -40,6 +41,11 @@ async def async_main(context): raise Exception("Apple notarization is enabled but apple_notarization_configs is not defined") setup_apple_notarization_credentials(context) + if "apple_hardened_signing" in all_signing_formats: + if not context.config.get("apple_signing_configs", False): + raise Exception("Apple signing is enabled but apple_signing_configs is not defined") + setup_apple_signing_credentials(context) + context.session = session context.autograph_configs = load_autograph_configs(context.config["autograph_configs"]) @@ -83,6 +89,14 @@ def get_default_config(base_dir=None): return default_config +def _write_text(path, contents): + with open(path, "wb") as fd: + if isinstance(contents, str): + fd.write(contents.encode("ascii")) + else: + fd.write(contents) + + def setup_apple_notarization_credentials(context): """Writes the notarization credential to a file @@ -93,6 +107,7 @@ def setup_apple_notarization_credentials(context): """ cert_type = task_cert_type(context) apple_notarization_configs = load_apple_notarization_configs(context.config["apple_notarization_configs"]) + if cert_type not in apple_notarization_configs: raise SigningScriptError("Credentials not found for scope: %s" % cert_type) scope_credentials = apple_notarization_configs.get(cert_type) @@ -101,15 +116,60 @@ def setup_apple_notarization_credentials(context): context.apple_credentials_path = os.path.join( os.path.dirname(context.config["apple_notarization_configs"]), - 'apple_api_key.json', + "apple_api_key.json", ) if os.path.exists(context.apple_credentials_path): # TODO: If we have different api keys for each product, this needs to overwrite every task: return # Convert dataclass to dict so json module can read it credential = asdict(scope_credentials[0]) - with open(context.apple_credentials_path, 'wb') as credfile: - credfile.write(json.dumps(credential).encode("ascii")) + _write_text(context.apple_credentials_path, json.dumps(credential)) + + +def setup_apple_signing_credentials(context): + """Writes the signing p12 file and password to a file + + Adds properties to context: apple_credentials_path + apple_credentials_pass_path + + Args: + context: Running task Context + """ + cert_type = task_cert_type(context) + + apple_signing_configs = load_apple_signing_configs(context.config["apple_signing_configs"]) + if cert_type not in apple_signing_configs: + raise SigningScriptError("Credentials not found for scope: %s" % cert_type) + scope_credentials = apple_signing_configs.get(cert_type) + if len(scope_credentials) != 1: + raise SigningScriptError("There should only be 1 scope credential, %s found." % len(scope_credentials)) + + context.apple_app_signing_pkcs12_path = os.path.join( + os.path.dirname(context.config["apple_signing_configs"]), + "apple_app_signing_creds.p12", + ) + unlink(context.apple_app_signing_pkcs12_path) + context.apple_installer_signing_pkcs12_path = os.path.join( + os.path.dirname(context.config["apple_signing_configs"]), + "apple_installer_signing_creds.p12", + ) + unlink(context.apple_installer_signing_pkcs12_path) + context.apple_signing_pkcs12_pass_path = os.path.join( + os.path.dirname(context.config["apple_signing_configs"]), + "apple_signing_creds_pass.passwd", + ) + unlink(context.apple_signing_pkcs12_pass_path) + + # Convert dataclass to dict so json module can read it + creds_config = asdict(scope_credentials[0]) + _write_text(context.apple_app_signing_pkcs12_path, base64.b64decode(creds_config["app_pkcs12_bundle"])) + + # Defaults to using the app credentials (ie: on Try) + if creds_config.get("installer_pkcs12_bundle"): + _write_text(context.apple_installer_signing_pkcs12_path, base64.b64decode(creds_config["installer_pkcs12_bundle"])) + else: + context.apple_installer_signing_pkcs12_path = context.apple_app_signing_pkcs12_path + + _write_text(context.apple_signing_pkcs12_pass_path, creds_config["pkcs12_password"]) def main(): diff --git a/signingscript/src/signingscript/sign.py b/signingscript/src/signingscript/sign.py index aadb1a8aa..f16ce5f42 100644 --- a/signingscript/src/signingscript/sign.py +++ b/signingscript/src/signingscript/sign.py @@ -30,9 +30,10 @@ from winsign.crypto import load_pem_certs from signingscript import task, utils +from signingscript.apple import copy_provisioning_profiles from signingscript.createprecomplete import generate_precomplete from signingscript.exceptions import SigningScriptError -from signingscript.rcodesign import RCodesignError, rcodesign_notarize, rcodesign_notary_wait, rcodesign_staple +from signingscript.rcodesign import RCodesignError, rcodesign_notarize, rcodesign_notary_wait, rcodesign_sign, rcodesign_staple log = logging.getLogger(__name__) @@ -1510,14 +1511,6 @@ async def sign_debian_pkg(context, path, fmt, *args, **kwargs): return path -def _can_notarize(filename, supported_extensions): - """ - Check if file can be notarized based on extension - """ - _, extension = os.path.splitext(filename) - return extension in supported_extensions - - async def _notarize_single(path, creds_path, staple=True): """Notarizes a single app/pkg retrying if necessary""" ATTEMPTS = 5 @@ -1553,7 +1546,7 @@ async def _notarize_pkg(context, path, workdir): workdir_files = os.listdir(workdir) # Filter supported file extensions - supported_files = [filename for filename in workdir_files if _can_notarize(filename, (".pkg",))] + supported_files = [filename for filename in workdir_files if filename.endswith(".pkg")] if not supported_files: raise SigningScriptError("No supported files found") @@ -1587,7 +1580,7 @@ async def _notarize_all(context, path, workdir): # Filter supported file extensions # We also support .pkg in case it's a tarball with .app + .pkg inside - supported_files = [filename for filename in workdir_files if _can_notarize(filename, (".app", ".pkg"))] + supported_files = [filename for filename in workdir_files if filename.endswith((".app", ".pkg"))] if not supported_files: raise SigningScriptError("No supported files found") @@ -1615,8 +1608,7 @@ async def apple_notarize(context, path, *args, **kwargs): shutil.rmtree(notarization_workdir, ignore_errors=True) utils.mkdir(notarization_workdir) - _, extension = os.path.splitext(path) - if extension == ".pkg": + if path.endswith(".pkg"): return await _notarize_pkg(context, path, notarization_workdir) else: return await _notarize_all(context, path, notarization_workdir) @@ -1633,3 +1625,72 @@ async def apple_notarize_geckodriver(context, path, *args, **kwargs): utils.mkdir(notarization_workdir) return await _notarize_geckodriver(context, path, notarization_workdir) + + +@time_async_function +async def apple_app_hardened_sign(context, path, *args, **kwargs): + """ + Sign an app using rcodesign. + """ + # Setup workdir + signing_dir = os.path.join(context.config["work_dir"], "extracted") + shutil.rmtree(signing_dir, ignore_errors=True) + utils.mkdir(signing_dir) + + if not path.endswith((".dmg", ".tar.gz")): + raise SigningScriptError("File format not supported.") + + extension = os.path.splitext(path)[-1] + # For now, we convert to tar, then continue work as tar + if extension == ".dmg": + await utils.extract_dmg(context, path, signing_dir) + else: + await _extract_tarfile(context, path, extension, signing_dir) + + # Get configs from task payload + payload = context.task.get("payload", {}) + hardened_sign_config = payload.get("hardened-sign-config") + assert hardened_sign_config and isinstance(hardened_sign_config, list) + provisioning_profile_config = payload.get("provisioning-profile-config") + + signed = False + for file in os.scandir(signing_dir): + if file.is_dir() and file.name.endswith(".app"): + # Developer ID Application certificate + creds = context.apple_app_signing_pkcs12_path + elif file.is_file() and file.name.endswith(".pkg"): + # Use installer credentials + creds = context.apple_installer_signing_pkcs12_path + else: + # If not pkg AND not a directory (.app) - then skip file + log.info(f"Skipping unsupported file at root: {file.path}") + continue + + bundle_path = os.path.join(signing_dir, file.path) + if provisioning_profile_config: + copy_provisioning_profiles(bundle_path, provisioning_profile_config) + + # TODO: widevine and omnija signing should run from formats? + for f in glob.glob(os.path.join(bundle_path, "**", "omni.ja")): + sign_omnija_with_autograph(context, f) + widevine_files = _get_widevine_signing_files(glob.glob(os.path.join(bundle_path, "**", "*"))) + for f, fmt in widevine_files.items(): # We should make _get_widevine_signing_files return a list + sign_widevine_with_autograph(context, f, "blessed" in fmt) + await rcodesign_sign( + context.config["work_dir"], + bundle_path, + creds, + context.apple_signing_pkcs12_pass_path, + hardened_sign_config, + ) + signed = True + if not signed: + raise SigningScriptError("Could not find an app to sign!") + # get all files from extracted folder (account for background, etc) + all_files = glob.glob(os.path.join(signing_dir, "**", "*"), recursive=True) + glob.glob(os.path.join(signing_dir, ".*")) + # filter for *non folders* so _create_tarfile doesn't include files multiple times + all_files = set([f for f in all_files if not os.path.isdir(f)]) + target = os.path.join(context.config["work_dir"], "public/build", "target.tar.gz") + if not os.path.exists(os.path.dirname(target)): + os.mkdir(os.path.dirname(target)) + return await _create_tarfile(context, target, all_files, "gz", signing_dir) diff --git a/signingscript/src/signingscript/task.py b/signingscript/src/signingscript/task.py index 493e3ebc3..266a8c36b 100644 --- a/signingscript/src/signingscript/task.py +++ b/signingscript/src/signingscript/task.py @@ -15,6 +15,7 @@ from scriptworker.utils import get_single_item_from_sequence from signingscript.sign import ( + apple_app_hardened_sign, apple_notarize, apple_notarize_geckodriver, sign_authenticode, @@ -39,6 +40,7 @@ "gpg": sign_gpg, "autograph_gpg": sign_gpg_with_autograph, "macapp": sign_macapp, + "apple_hardened_signing": apple_app_hardened_sign, "widevine": sign_widevine, "autograph_debsign": sign_debian_pkg, "autograph_widevine": sign_widevine, diff --git a/signingscript/src/signingscript/utils.py b/signingscript/src/signingscript/utils.py index 5ca757f69..26e51d3bf 100644 --- a/signingscript/src/signingscript/utils.py +++ b/signingscript/src/signingscript/utils.py @@ -28,11 +28,21 @@ class Autograph: @dataclass class AppleNotarization: """Apple notarization configuration object.""" + issuer_id: str key_id: str private_key: str +@dataclass +class AppleSigning: + """Apple signing configuration object.""" + + app_pkcs12_bundle: str + installer_pkcs12_bundle: str + pkcs12_password: str + + def mkdir(path): """Equivalent to `mkdir -p`. @@ -47,6 +57,20 @@ def mkdir(path): pass +def unlink(path): + """Equivalent to rm -f {file} + Ignores FileNotFound errors (as rm -f would) + + Args: + path (str): the path to remove + """ + try: + os.unlink(path) + log.info(f"removed {path}") + except FileNotFoundError: + pass + + def get_hash(path, hash_type="sha512"): """Get the hash of a given path. @@ -97,7 +121,7 @@ def _load_scoped_configs(filename, cls, name): for scope, config in raw_cfg.items(): if cls == Autograph: scope_configs[scope] = [cls(*s) for s in config] - elif cls == AppleNotarization: + elif cls in (AppleNotarization, AppleSigning): scope_configs[scope] = [cls(**s) for s in config] else: raise SigningScriptError("Unknown class for scoped configs: %s" % cls.__name__) @@ -131,6 +155,19 @@ def load_apple_notarization_configs(filename): return _load_scoped_configs(filename, AppleNotarization, "Apple Notarization") +def load_apple_signing_configs(filename): + """Load the apple notarization configuration from `filename`. + + Args: + filename (str): config file + + Returns: + dict of Apple Notarization objects: keyed by signing cert type + + """ + return _load_scoped_configs(filename, AppleSigning, "Apple Signing") + + async def log_output(fh, log_level=logging.INFO): """Log the output from an async generator. @@ -238,3 +275,22 @@ def split_autograph_format(format_): return format_.split(":", 1) else: return format_, None + + +async def extract_dmg(context, source, dest): + """Extracts a source DMG into dest folder + + Args: + context: The running task context object + source (str): the DMG sorce path + dest (str): the destination directory to extract files + """ + dmg = context.config["dmg"] + hfsplus = context.config["hfsplus"] + undmg_cmd = [dmg, "extract", source, "tmp.hfs"] + log.info(undmg_cmd) + await execute_subprocess(undmg_cmd, cwd=context.config["work_dir"], log_level=logging.DEBUG) + + hfsplus_cmd = [hfsplus, "tmp.hfs", "extractall", "/", dest] + log.info(hfsplus_cmd) + await execute_subprocess(hfsplus_cmd, cwd=context.config["work_dir"], log_level=logging.DEBUG) diff --git a/signingscript/tests/conftest.py b/signingscript/tests/conftest.py index df7844961..79cd744dd 100644 --- a/signingscript/tests/conftest.py +++ b/signingscript/tests/conftest.py @@ -19,6 +19,7 @@ def read_file(path): BASE_DIR = os.path.dirname(os.path.dirname(__file__)) SERVER_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "example_server_config.json") APPLE_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "example_apple_notarization_config.json") +APPLE_SIGNING_CONFIG_PATH = os.path.join(os.path.dirname(__file__), "example_apple_signing_config.json") DEFAULT_SCOPE_PREFIX = "project:releng:signing:" TEST_CERT_TYPE = f"{DEFAULT_SCOPE_PREFIX}cert:dep-signing" TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") @@ -55,8 +56,12 @@ def context(tmpdir): context.config["artifact_dir"] = os.path.join(tmpdir, "artifact") context.config["taskcluster_scope_prefixes"] = [DEFAULT_SCOPE_PREFIX] context.config["apple_notarization_configs"] = APPLE_CONFIG_PATH + context.config["apple_signing_configs"] = APPLE_CONFIG_PATH context.autograph_configs = load_autograph_configs(SERVER_CONFIG_PATH) - context.apple_credentials_path = "fakepath" + context.apple_credentials_path = os.path.join(tmpdir, "fakepath") + context.apple_app_signing_pkcs12_path = os.path.join(tmpdir, "apple_app.p12") + context.apple_installer_signing_pkcs12_path = os.path.join(tmpdir, "apple_installer.p12") + context.apple_signing_creds_path = os.path.join(tmpdir, "apple_p12.passwd") mkdir(context.config["work_dir"]) mkdir(context.config["artifact_dir"]) context.task = {"scopes": [TEST_CERT_TYPE]} diff --git a/signingscript/tests/data/test.dmg b/signingscript/tests/data/test.dmg new file mode 100644 index 000000000..2970bc111 Binary files /dev/null and b/signingscript/tests/data/test.dmg differ diff --git a/signingscript/tests/example_apple_signing_config.json b/signingscript/tests/example_apple_signing_config.json new file mode 100644 index 000000000..cedc23862 --- /dev/null +++ b/signingscript/tests/example_apple_signing_config.json @@ -0,0 +1,7 @@ +{ + "project:releng:signing:cert:dep-signing": [{ + "app_pkcs12_bundle": "abcdef", + "installer_pkcs12_bundle": "abcdef", + "pkcs12_password": "verysecret" + }] +} diff --git a/signingscript/tests/test_config.py b/signingscript/tests/test_config.py index ff0b48d07..7f190abba 100644 --- a/signingscript/tests/test_config.py +++ b/signingscript/tests/test_config.py @@ -12,6 +12,7 @@ "PUBLIC_IP": "0.0.0.0", "PASSWORDS_PATH": "", "APPLE_NOTARIZATION_CREDS_PATH": "", + "APPLE_SIGNING_CONFIG_PATH": "", "SSL_CERT_PATH": "", "SIGNTOOL_PATH": "", "DMG_PATH": "", diff --git a/signingscript/tests/test_rcodesign.py b/signingscript/tests/test_rcodesign.py index e254b80e5..1ed0f80b5 100644 --- a/signingscript/tests/test_rcodesign.py +++ b/signingscript/tests/test_rcodesign.py @@ -6,6 +6,8 @@ import pytest from conftest import TEST_DATA_DIR +from pathlib import Path + import signingscript.rcodesign as rcodesign @@ -142,14 +144,14 @@ async def test_find_submission_id_fail(): def mock_async_generator(*values): vals = [*values] - async def func(): + async def func(*args, **kwargs): return vals.pop(0) return func def mock_sync_generator(*values): vals = [*values] - def func(): + def func(*args, **kwargs): return vals.pop(0) return func @@ -167,3 +169,44 @@ async def mock_create_subprocess_exec(*args, **kwargs): assert exitcode == 96 assert len(output) > 1 assert output == ["hello", "err"] + + +@pytest.mark.asyncio +async def test_rcodesign_sign(context, mocker): + + workdir = Path(context.config["work_dir"]) + app_path = workdir / "test.app" + app_path.mkdir() + (app_path / "samplefile").touch() + (app_path / "samplefile2").touch() + context.apple_app_signing_pkcs12_path = workdir / "test_cred.p12" + context.apple_signing_pkcs12_pass_path = workdir / "test_cred.passwd" + entitlement_file = workdir / "test.xml" + entitlement_file.touch() + + hardened_sign_config = [ + { + "entitlements": "https://fakeurl", + "runtime": True, + "globs": [ + "/*" + ] + } + ] + + download = mock.AsyncMock() + download.side_effect = [{"https://fakeurl": str(entitlement_file)}] + mocker.patch.object(rcodesign, "_download_entitlements", download) + execute = mock.AsyncMock() + execute.side_effect = lambda x: x + mocker.patch.object(rcodesign, "_execute_command", execute) + + await rcodesign.rcodesign_sign( + context.config["work_dir"], + str(app_path), + context.apple_app_signing_pkcs12_path, + context.apple_signing_pkcs12_pass_path, + hardened_sign_config, + ) + download.assert_called_once() + execute.assert_called_once() diff --git a/signingscript/tox.ini b/signingscript/tox.ini index 8f458eaae..3daba081d 100644 --- a/signingscript/tox.ini +++ b/signingscript/tox.ini @@ -14,14 +14,19 @@ commands = [testenv] usedevelop = true depends = clean +skip_install = true setenv = PYTHONDONTWRITEBYTECODE=1 - PYTHONPATH = {toxinidir}/tests + PYTHONPATH = {toxinidir}/tests:{toxinidir}/../scriptworker_client/src deps= py38: -r requirements/test.py38.txt + -r {toxinidir}/../scriptworker_client/requirements/test.py38.txt py39: -r requirements/test.txt + -r {toxinidir}/../scriptworker_client/requirements/test.txt commands = + python -I -m pip install {toxinidir}/../scriptworker_client + python -I -m pip install -e {toxinidir} {posargs:py.test --cov-config=tox.ini --cov-append --cov={toxinidir}/src/signingscript --cov-report term-missing tests} [testenv:clean] diff --git a/tox.ini b/tox.ini index 35c177cb6..bfa1bccbb 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,6 @@ envlist = scriptworker_client-py39 shipitscript-py38 shipitscript-py39 - signingscript-py38 signingscript-py39 treescript-py38 treescript-py39 @@ -193,11 +192,6 @@ changedir = {toxinidir}/shipitscript commands = tox -e py39 -[testenv:signingscript-py38] -changedir = {toxinidir}/signingscript -commands = - tox -e py38 - [testenv:signingscript-py39] changedir = {toxinidir}/signingscript commands = @@ -218,6 +212,6 @@ commands = [testenv:ruff-py39] deps = ruff -commands = +commands = ruff --version - ruff --verbose {toxinidir} \ No newline at end of file + ruff --verbose {toxinidir}