From b30ef3726e2ca1e134ed11f7769befd0d7b98dea Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 28 Jul 2023 11:46:15 +1000 Subject: [PATCH 01/31] feat: enable repo finder to support more languages via Open Source Insights Signed-off-by: Ben Selwyn-Smith --- docs/source/pages/using.rst | 16 +- src/macaron/config/defaults.ini | 7 +- .../dependency_resolver.py | 26 -- .../dependency_analyzer/java_repo_finder.py | 306 ----------------- src/macaron/repo_finder/__init__.py | 4 + src/macaron/repo_finder/repo_finder.py | 51 +++ src/macaron/repo_finder/repo_finder_base.py | 87 +++++ src/macaron/repo_finder/repo_finder_dd.py | 203 +++++++++++ src/macaron/repo_finder/repo_finder_java.py | 323 ++++++++++++++++++ src/macaron/slsa_analyzer/analyzer.py | 53 ++- .../__init__.py | 0 tests/repo_finder/repo_finder_dd/__init__.py | 2 + .../repo_finder_dd/test_repo_finder_dd.py | 16 + .../repo_finder/repo_finder_java/__init__.py | 2 + .../resources/example_pom.xml | 0 .../resources/example_pom_no_scm.xml | 0 .../test_repo_finder_java.py} | 18 +- 17 files changed, 761 insertions(+), 353 deletions(-) delete mode 100644 src/macaron/dependency_analyzer/java_repo_finder.py create mode 100644 src/macaron/repo_finder/__init__.py create mode 100644 src/macaron/repo_finder/repo_finder.py create mode 100644 src/macaron/repo_finder/repo_finder_base.py create mode 100644 src/macaron/repo_finder/repo_finder_dd.py create mode 100644 src/macaron/repo_finder/repo_finder_java.py rename tests/{dependency_analyzer/java_repo_finder => repo_finder}/__init__.py (100%) create mode 100644 tests/repo_finder/repo_finder_dd/__init__.py create mode 100644 tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py create mode 100644 tests/repo_finder/repo_finder_java/__init__.py rename tests/{dependency_analyzer/java_repo_finder => repo_finder/repo_finder_java}/resources/example_pom.xml (100%) rename tests/{dependency_analyzer/java_repo_finder => repo_finder/repo_finder_java}/resources/example_pom_no_scm.xml (100%) rename tests/{dependency_analyzer/java_repo_finder/test_java_repo_finder.py => repo_finder/repo_finder_java/test_repo_finder_java.py} (74%) diff --git a/docs/source/pages/using.rst b/docs/source/pages/using.rst index 7fd7909b4..b424d1a83 100644 --- a/docs/source/pages/using.rst +++ b/docs/source/pages/using.rst @@ -203,13 +203,15 @@ This feature is enabled by default. To disable, or configure its behaviour in ot See :ref:`dump-defaults `, the CLI command to dump the default configurations in ``defaults.ini``. After making changes, see :ref:`analyze ` CLI command for the option to pass the modified ``defaults.ini`` file. -Within the configuration file under the ``repofinder.java`` header, five options exist: ``find_repos``, ``artifact_repositories``, ``repo_pom_paths``, ``find_parents``, ``artifact_ignore_list``. These options behave as follows: +Within the configuration file under the ``repofinder.java`` header, three options exist: ``artifact_repositories``, ``repo_pom_paths``, ``find_parents``. These options behave as follows: -- ``find_repos`` (Values: True or False) - Enables or disables the Repository Finding feature. - ``artifact_repositories`` (Values: List of URLs) - Determines the remote artifact repositories to attempt to retrieve dependency information from. - ``repo_pom_paths`` (Values: List of POM tags) - Determines where to search for repository information in the POM files. E.g. scm.url. - ``find_parents`` (Values: True or False) - When enabled, the Repository Finding feature will also search for repository URLs in parents POM files of the current dependency. -- ``artifact_ignore_list`` (Values: List of GAs) - The Repository Finding feature will skip any artifact in this list. Format is "GroupId":"ArtifactId". E.g. org.apache.maven:maven + +The entire feature can be disabled via the ``find_repos`` option found under the configuration header ``repofinder``, as so: + +- ``find_repos`` (Values: True or False) - Enables or disables the Repository Finding feature. .. note:: Finding repositories requires at least one remote call, adding some additional overhead to an analysis run. @@ -217,16 +219,18 @@ An example configuration file for utilising this feature: .. code-block:: ini - [repofinder.java] + [repofinder] find_repos = True + + [repofinder.java] artifact_repositories = https://repo.maven.apache.org/maven2 repo_pom_paths = scm.url scm.connection scm.developerConnection find_parents = True - artifact_ignore_list = - org.apache.maven:maven + + ------------------------------------- Analyzing a locally cloned repository diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index 5710a19b8..fc67d129c 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -44,19 +44,18 @@ timeout = 2400 recursive = False # This is the repo finder script. +[repofinder] +find_repos = True + [repofinder.java] # The list of maven-like repositories to attempt to retrieve artifact POMs from. artifact_repositories = https://repo.maven.apache.org/maven2 -find_repos = True repo_pom_paths = scm.url scm.connection scm.developerConnection find_parents = True parent_limit = 10 -# Disables repo finding for specific artifacts based on their group and artifact IDs. Format: {groupId}:{artifactId} -# E.g. com.oracle.coherence.ce:coherence -artifact_ignore_list = # Git services that Macaron has access to clone repositories. # For security purposes, Macaron will only clone repositories from the hostnames specified. diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index 1a16d3f49..6de1d0888 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -11,9 +11,7 @@ from packaging import version -from macaron.config.defaults import defaults from macaron.config.target_config import Configuration -from macaron.dependency_analyzer.java_repo_finder import find_java_repo from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus from macaron.slsa_analyzer.git_url import get_remote_vcs_url, get_repo_full_name_from_url @@ -133,9 +131,6 @@ def add_latest_version( url_to_artifact: dict[str, set] Used to detect artifacts that have similar repos. """ - if defaults.getboolean("repofinder.java", "find_repos"): - DependencyAnalyzer._find_repo(item) - # Check if the URL is already seen for a different artifact. if item["url"] != "": artifacts = url_to_artifact.get(item["url"]) @@ -173,27 +168,6 @@ def add_latest_version( except ValueError as error: logger.error("Could not parse dependency version number: %s", error) - @staticmethod - def _find_repo(item: DependencyInfo) -> None: - """Find the repo for the current item, if the criteria are met.""" - if item["url"] != "" or item["version"] == "unspecified" or not item["group"] or not item["name"]: - logger.debug("Item URL already exists, or item is missing information: %s", item) - return - gav = f"{item['group']}:{item['name']}:{item['version']}" - if f"{item['group']}:{item['name']}" in defaults.get_list("repofinder.java", "artifact_ignore_list"): - logger.debug("Skipping GAV: %s", gav) - return - - urls = find_java_repo( - item["group"], - item["name"], - item["version"], - defaults.get_list("repofinder.java", "repo_pom_paths"), - ) - item["url"] = DependencyAnalyzer.find_valid_url(list(urls)) - if item["url"] == "": - logger.debug("Failed to find url for GAV: %s", gav) - @staticmethod def find_valid_url(urls: Iterable[str]) -> str: """Find a valid URL from the provided URLs. diff --git a/src/macaron/dependency_analyzer/java_repo_finder.py b/src/macaron/dependency_analyzer/java_repo_finder.py deleted file mode 100644 index 4ec76fb6f..000000000 --- a/src/macaron/dependency_analyzer/java_repo_finder.py +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. - -"""This module tries to find urls of repositories that match artifacts passed in 'group:artifact:version' form.""" -import logging -import re -from collections.abc import Iterator -from xml.etree.ElementTree import Element # nosec - -import defusedxml.ElementTree -import requests -from defusedxml.ElementTree import fromstring - -from macaron.config.defaults import defaults - -logger: logging.Logger = logging.getLogger(__name__) - - -def create_urls(group: str, artifact: str, version: str, repositories: list[str]) -> list[str]: - """ - Create the urls to search for the pom relating to the passed GAV. - - Parameters - ---------- - group : str - The group ID. - artifact: str - The artifact ID. - version: str - The version of the artifact. - repositories: list[str] - The list of repository URLs to use as the base for the new URLs. - - Returns - ------- - list[str] - The list of created URLs. - """ - urls = [] - for repo in repositories: - urls.append(f"{repo}/{group}/{artifact}/{version}/{artifact}-{version}.pom") - return urls - - -def retrieve_pom(session: requests.Session, url: str) -> str: - """ - Attempt to retrieve the file located at the passed URL using the passed Session. - - Parameters - ---------- - session : requests.Session - The HTTP session to use for attempting the GET request. - url : str - The URL for the GET request. - - Returns - ------- - str : - The retrieved file data or an empty string. - """ - if not url.endswith(".pom"): - return "" - - try: - res = session.get(url) - except (requests.RequestException, OSError) as error: - logger.debug("Error during pom retrieval: %s", error) - return "" - - if not res.ok: - logger.debug("Failed to retrieve pom from: %s, error code: %s", url, res.status_code) - return "" - - logger.debug("Found artifact POM at: %s", url) - - return res.text - - -def _find_element(parent: Element | None, target: str) -> Element | None: - if not parent: - return None - - # Attempt to match the target tag within the children of parent. - for child in parent: - # Account for raw tags, and tags accompanied by Maven metadata enclosed in curly braces. E.g. '{metadata}tag' - if child.tag == target or child.tag.endswith(f"}}{target}"): - return child - return None - - -def find_parent(pom: Element) -> tuple[str, str, str]: - """ - Extract parent information from passed POM. - - Parameters - ---------- - pom : str - The POM as a string. - - Returns - ------- - tuple[str] : - The GAV of the parent artifact. - """ - element = _find_element(pom, "parent") - if element is None: - return "", "", "" - group = _find_element(element, "groupId") - artifact = _find_element(element, "artifactId") - version = _find_element(element, "version") - if ( - group is not None - and group.text - and artifact is not None - and artifact.text - and version is not None - and version.text - ): - return group.text.strip(), artifact.text.strip(), version.text.strip() - return "", "", "" - - -def find_scm(pom: Element, tags: list[str], resolve_properties: bool = True) -> tuple[Iterator[str], int]: - """ - Parse the passed pom and extract the passed tags. - - Parameters - ---------- - pom : Element - The parsed POM. - tags : list[str] - The list of tags to try extracting from the POM. - resolve_properties: bool - Whether to attempt resolution of Maven properties within the POM. - - Returns - ------- - tuple[Iterator[str], int] : - The extracted contents of any matches tags, and the number of matches, as a tuple. - """ - results = [] - - # Try to match each tag with the contents of the POM. - for tag in tags: - element: Element | None = pom - - if tag.startswith("properties."): - # Tags under properties are often "." separated - # These can be safely split into two resulting tags as nested tags are not allowed here - tag_parts = ["properties", tag[11:]] - else: - # Other tags can be split into distinct elements via "." - tag_parts = tag.split(".") - - for index, tag_part in enumerate(tag_parts): - element = _find_element(element, tag_part) - if element is None: - break - if index == len(tag_parts) - 1 and element.text: - # Add the contents of the final tag - results.append(element.text.strip()) - - # Resolve any Maven properties within the results - if resolve_properties: - results = _resolve_properties(pom, results) - - return iter(results), len(results) - - -def _resolve_properties(pom: Element, values: list[str]) -> list[str]: - """Resolve any Maven properties found within the passed list of values. - - Maven POM files have five different use cases for properties (see https://maven.apache.org/pom.html). - Only the two that relate to contents found elsewhere within the same POM file are considered here. - That is: ${project.x} where x can be a child tag at any depth, or ${x} where x is found at project.properties.x. - Entries with unresolved properties are not included in the returned list. In the case of chained properties, - only the top most property is evaluated. - """ - resolved_values = [] - for value in values: - replacements: list = [] - # Calculate replacements - matches any number of ${...} entries in the current value - for match in re.finditer("\\$\\{[^}]+}", value): - text = match.group().replace("$", "").replace("{", "").replace("}", "") - if text.startswith("project."): - text = text.replace("project.", "") - else: - text = f"properties.{text}" - # Call find_scm with property resolution flag set to False to prevent the possibility of endless looping - value_iterator, count = find_scm(pom, [text], False) - if count == 0: - break - replacements.append([match.start(), next(value_iterator), match.end()]) - - # Apply replacements in reverse order - # E.g. - # git@github.com:owner/project${javac.src.version}-${project.inceptionYear}.git - # -> - # git@github.com:owner/project${javac.src.version}-2023.git - # -> - # git@github.com:owner/project1.8-2023.git - for replacement in reversed(replacements): - value = f"{value[:replacement[0]]}{replacement[1]}{value[replacement[2]:]}" - - resolved_values.append(value) - - return resolved_values - - -def parse_pom(pom: str) -> Element | None: - """ - Parse the passed POM using defusedxml. - - Parameters - ---------- - pom : str - The contents of a POM file as a string. - - Returns - ------- - Element | None : - The parsed element representing the POM's XML hierarchy. - """ - try: - pom_element: Element = fromstring(pom) - return pom_element - except defusedxml.ElementTree.ParseError as error: - logger.debug("Failed to parse XML: %s", error) - return None - - -def find_java_repo(group: str, artifact: str, version: str, tags: list[str]) -> Iterator[str]: - """ - Attempt to retrieve a repository URL that matches the passed GAV artifact. - - Parameters - ---------- - group : str - The group identifier of an artifact. - artifact : str - The artifact name of an artifact. - version : str - The version number of an artifact. - tags : Iterator[str] - The list of XML tags to look for, each in the format: tag1[.tag2 ... .tagN]. - - Yields - ------ - Iterator[str] : - The URLs found for the passed GAV. - """ - repositories = defaults.get_list( - "repofinder.java", "artifact_repositories", fallback=["https://repo.maven.apache.org/maven2"] - ) - if not any(tags): - logger.debug("No POM tags found for URL discovery.") - return - - # Perform the following in a loop: - # - Create URLs for the current artifact POM - # - Parse the POM - # - Try to extract SCM metadata and return URLs - # - Try to extract parent information and change current artifact to it - # - Repeat - limit = defaults.getint("repofinder.java", "parent_limit", fallback=10) - while group and artifact and version and limit > 0: - # Create the URLs for retrieving the artifact's POM - group = group.replace(".", "/") - request_urls = create_urls(group, artifact, version, repositories) - if not request_urls: - # Abort if no URLs were created - return - - # Try each POM URL in order, terminating early if a match is found - with requests.Session() as session: - pom = "" - for request_url in request_urls: - pom = retrieve_pom(session, request_url) - if pom != "": - break - - if pom == "": - # Abort if no POM was found - return - - # Parse POM using defusedxml - pom_element = parse_pom(pom) - if pom_element is None: - return - - # Attempt to extract SCM data and return URL - urls, url_count = find_scm(pom_element, tags) - - if url_count > 0: - yield from urls - - if defaults.getboolean("repofinder.java", "find_parents"): - # Attempt to extract parent information from POM - group, artifact, version = find_parent(pom_element) - else: - break - - limit = limit - 1 - - # Nothing found - return diff --git a/src/macaron/repo_finder/__init__.py b/src/macaron/repo_finder/__init__.py new file mode 100644 index 000000000..c406a64cc --- /dev/null +++ b/src/macaron/repo_finder/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This package contains the dependency resolvers for Java projects.""" diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py new file mode 100644 index 000000000..f65ccb1d7 --- /dev/null +++ b/src/macaron/repo_finder/repo_finder.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the logic for using/calling the different repo finders.""" + +import logging +from collections.abc import Iterator + +from packageurl import PackageURL + +from macaron.repo_finder.repo_finder_base import BaseRepoFinder +from macaron.repo_finder.repo_finder_dd import RepoFinderDD +from macaron.repo_finder.repo_finder_java import JavaRepoFinder + +logger: logging.Logger = logging.getLogger(__name__) + + +def find_repo(purl_string: str) -> Iterator[str]: + """Retrieve the repository URL that matches the given PURL. + + Parameters + ---------- + purl_string : str + The purl string representing a package. + + Yields + ------ + Iterator[str] : + The repository URLs found for the passed package. + """ + # Parse the purl string + try: + purl = PackageURL.from_string(purl_string) + except ValueError as error: + logger.debug("Invalid PURL: %s, %s", purl_string, error) + return + + repo_finder: BaseRepoFinder + + match purl.type: + case "maven": + repo_finder = JavaRepoFinder() + case "pypi": + repo_finder = RepoFinderDD(purl.type) + + case _: + logger.debug("Unsupported type in PURL: %s (%s)", purl.type, purl_string) + return + + logger.debug("Analyzing %s with Repo Finder: %s", purl_string, repo_finder.__class__) + yield from repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py new file mode 100644 index 000000000..100694113 --- /dev/null +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -0,0 +1,87 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the base class for the repo finders.""" + +from abc import ABC, abstractmethod +from collections.abc import Iterator + +import requests + + +class BaseRepoFinder(ABC): + """This abstract class is used to represent Repository Finders.""" + + @abstractmethod + def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + """ + Attempt to retrieve a repository URL that matches the passed artifact. + + Parameters + ---------- + group : str + The group identifier of an artifact. + artifact : str + The artifact name of an artifact. + version : str + The version number of an artifact. + + Yields + ------ + Iterator[str] : + The URLs found for the passed GAV. + """ + + @abstractmethod + def create_urls(self, group: str, artifact: str, version: str) -> list[str]: + """ + Create the urls to search for the metadata relating to the passed artifact. + + Parameters + ---------- + group : str + The group ID. + artifact: str + The artifact ID. + version: str + The version of the artifact. + + Returns + ------- + list[str] + The list of created URLs. + """ + + @abstractmethod + def retrieve_metadata(self, session: requests.Session, url: str) -> str: + """ + Attempt to retrieve the file located at the passed URL using the passed Session. + + Parameters + ---------- + session : requests.Session + The HTTP session to use for attempting the GET request. + url : str + The URL for the GET request. + + Returns + ------- + str : + The retrieved file data or an empty string. + """ + + @abstractmethod + def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + """ + Parse the passed metadata and extract the relevant information. + + Parameters + ---------- + metadata : str + The metadata as a string. + + Returns + ------- + tuple[Iterator[str], int] : + The extracted contents in iterable form and count as a tuple. + """ diff --git a/src/macaron/repo_finder/repo_finder_dd.py b/src/macaron/repo_finder/repo_finder_dd.py new file mode 100644 index 000000000..254309d5c --- /dev/null +++ b/src/macaron/repo_finder/repo_finder_dd.py @@ -0,0 +1,203 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the PythonRepoFinderDD class to be used for finding repositories using deps.dev.""" +import json +import logging +from collections.abc import Iterator +from urllib.parse import quote as encode + +import requests + +from macaron.repo_finder.repo_finder_base import BaseRepoFinder + +logger: logging.Logger = logging.getLogger(__name__) + + +class RepoFinderDD(BaseRepoFinder): + """This class is used to find repositories using Google's Open Source Insights tool (deps.dev).""" + + # The label used by deps.dev to denote repository urls (Based on observation ONLY) + repo_url_label = "SOURCE_REPO" + + def __init__(self, purl_type: str) -> None: + """Initialise the deps.dev repository finder instance. + + Parameters + ---------- + purl_type : str + The PURL type this instance is intended for use with. + """ + self.type = purl_type + + def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + """ + Attempt to retrieve a repository URL that matches the passed artifact. + + Parameters + ---------- + group : str + The group identifier of an artifact. + artifact : str + The artifact name of an artifact. + version : str + The version number of an artifact. + + Yields + ------ + Iterator[str] : + The URLs found for the passed GAV. + """ + request_urls = self.create_urls(group, artifact, version) + if not request_urls: + logger.debug("No urls found for: %s", artifact) + return + + with requests.Session() as session: + metadata = self.retrieve_metadata(session, request_urls[0]) + if not metadata: + logger.debug("Failed to retrieve metadata for: %s", artifact) + return + + urls, url_count = self.read_metadata(metadata) + if url_count == 0: + logger.debug("Failed to extract repository URLs from metadata: %s", artifact) + return + + yield from urls + + def create_urls(self, group: str, artifact: str, version: str) -> list[str]: + """ + Create the urls to search for the metadata relating to the passed artifact. + + If a version is not specified, remote API calls will be used to try and find one. + + Parameters + ---------- + group : str + The group ID. + artifact: str + The artifact ID. + version: str + The version of the artifact. + + Returns + ------- + list[str] + The list of created URLs. + """ + package_name = f"{group}:{artifact}:{version}" + base_url = self.create_type_specific_url(group, artifact) + + if version: + return [f"{base_url}/versions/{version}"] + + # Find the latest version. + with requests.Session() as session: + try: + result = session.get(base_url) + except (requests.RequestException, OSError) as error: + logger.debug("Error during version retrieval: %s", error) + return [] + + if not result.ok: + logger.debug("Failed to retrieve versions for: %s, error code: %s", package_name, result.status_code) + return [] + + metadata = json.loads(result.text) + versions = metadata["versions"] + latest_version = versions[len(version) - 1]["versionKey"]["version"] + + if latest_version: + return [f"{base_url}/versions/{latest_version}"] + + return [] + + def retrieve_metadata(self, session: requests.Session, url: str) -> str: + """ + Attempt to retrieve the file located at the passed URL using the passed Session. + + Parameters + ---------- + session : requests.Session + The HTTP session to use for attempting the GET request. + url : str + The URL for the GET request. + + Returns + ------- + str : + The retrieved file data or an empty string. + """ + try: + result = session.get(url) + except (requests.RequestException, OSError) as error: + logger.debug("Error during metadata retrieval: %s", error) + return "" + + if not result.ok: + logger.debug("Failed to retrieve metadata at: %s, error code: %s", url, result.status_code) + return "" + + return result.text + + def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + """ + Parse the deps.dev metadata and extract the repository links. + + Parameters + ---------- + metadata : str + The metadata as a string. + + Returns + ------- + tuple[Iterator[str], int] : + The extracted contents in iterable form and count as a tuple. + """ + parsed = json.loads(metadata) + + if not parsed["links"]: + logger.debug("Metadata had no URLs: %s", parsed["versionKey"]) + return iter([]), 0 + + for link in parsed["links"]: + if link["label"] == self.repo_url_label: + return iter([link["url"]]), 1 + + return iter([]), 0 + + def create_type_specific_url(self, namespace: str, name: str) -> str: + """Create a url for the deps.dev API based on the package type. + + Parameters + ---------- + namespace : str + The PURL namespace element. + name : str + The PURL name element. + + Returns + ------- + str : + The specific URL relating to the package. + """ + namespace = encode(namespace) + name = encode(name) + + match self.type: + case "pypi": + package_name = name.lower().replace("_", "-") + case "npm": + if namespace: + package_name = f"%40{namespace}%2F{name}" + else: + package_name = name + case "nuget" | "cargo": + package_name = name + + case _: + logger.debug("PURL type not yet supported: %s", self.type) + return "" + + return f"https://api.deps.dev/v3alpha/systems/{self.type}/packages/{package_name}" diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py new file mode 100644 index 000000000..c5ac090aa --- /dev/null +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -0,0 +1,323 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the JavaRepoFinder class to be used for finding Java repositories.""" +import logging +import re +import typing +from collections.abc import Iterator +from xml.etree.ElementTree import Element # nosec + +import defusedxml.ElementTree +import requests +from defusedxml.ElementTree import fromstring + +from macaron.config.defaults import defaults +from macaron.repo_finder.repo_finder_base import BaseRepoFinder + +logger: logging.Logger = logging.getLogger(__name__) + + +class JavaRepoFinder(BaseRepoFinder): + """This class is used to find Java repositories.""" + + def __init__(self) -> None: + """Initialise the Java repository finder instance.""" + self.pom_element: Element | None = None + + def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + """ + Attempt to retrieve a repository URL that matches the passed artifact. + + Parameters + ---------- + group : str + The group identifier of an artifact. + artifact : str + The artifact name of an artifact. + version : str + The version number of an artifact. + + Yields + ------ + Iterator[str] : + The URLs found for the passed GAV. + """ + # Perform the following in a loop: + # - Create URLs for the current artifact POM + # - Parse the POM + # - Try to extract SCM metadata and return URLs + # - Try to extract parent information and change current artifact to it + # - Repeat + group = group.replace(".", "/") + limit = defaults.getint("repofinder.java", "parent_limit", fallback=10) + while group and artifact and version and limit > 0: + # Create the URLs for retrieving the artifact's POM + group = group.replace(".", "/") + request_urls = self.create_urls(group, artifact, version) + if not request_urls: + # Abort if no URLs were created + logger.debug("Failed to create request URLs for %s:%s:%s", group, artifact, version) + return + + # Try each POM URL in order, terminating early if a match is found + with requests.Session() as session: + pom = "" + for request_url in request_urls: + pom = self.retrieve_metadata(session, request_url) + if pom != "": + break + + if pom == "": + # Abort if no POM was found + logger.debug("No POM found for %s:%s:%s", group, artifact, version) + return + + urls, url_count = self.read_metadata(pom) + + if url_count > 0: + logger.debug("Found %s urls.", url_count) + yield from urls + + if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: + # Attempt to extract parent information from POM + group, artifact, version = self.find_parent(self.pom_element) + else: + break + + limit = limit - 1 + + # Nothing found + return + + def create_urls(self, group: str, artifact: str, version: str) -> list[str]: + """ + Create the urls to search for the metadata relating to the passed artifact. + + Parameters + ---------- + group : str + The group ID. + artifact: str + The artifact ID. + version: str + The version of the artifact. + + Returns + ------- + list[str] + The list of created URLs. + """ + repositories = defaults.get_list( + "repofinder.java", "artifact_repositories", fallback=["https://repo.maven.apache.org/maven2"] + ) + urls = [] + for repo in repositories: + urls.append(f"{repo}/{group}/{artifact}/{version}/{artifact}-{version}.pom") + return urls + + def retrieve_metadata(self, session: requests.Session, url: str) -> str: + """ + Attempt to retrieve the file located at the passed URL using the passed Session. + + Parameters + ---------- + session : requests.Session + The HTTP session to use for attempting the GET request. + url : str + The URL for the GET request. + + Returns + ------- + str : + The retrieved file data or an empty string. + """ + try: + res = session.get(url) + except (requests.RequestException, OSError) as error: + logger.debug("Error during pom retrieval: %s", error) + return "" + + if not res.ok: + logger.debug("Failed to retrieve pom from: %s, error code: %s", url, res.status_code) + return "" + + logger.debug("Found artifact POM at: %s", url) + + return res.text + + def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + """ + Parse the passed pom and extract the relevant tags. + + Parameters + ---------- + metadata : str + The metadata as a string. + + Returns + ------- + tuple[Iterator[str], int] : + The extracted contents in iterable form and count as a tuple. + """ + # Retrieve tags + tags = defaults.get_list("repofinder.java", "repo_pom_paths") + if not any(tags): + logger.debug("No POM tags found for URL discovery.") + return iter([]), 0 + + # Parse POM using defusedxml + pom_element = self.parse_pom(metadata) + if pom_element is None: + return iter([]), 0 + + # Attempt to extract SCM data and return URL + return self.find_scm(pom_element, tags) + + def parse_pom(self, pom: str) -> Element | None: + """ + Parse the passed POM using defusedxml. + + Parameters + ---------- + pom : str + The contents of a POM file as a string. + + Returns + ------- + Element | None : + The parsed element representing the POM's XML hierarchy. + """ + try: + self.pom_element = fromstring(pom) + return self.pom_element + except defusedxml.ElementTree.ParseError as error: + logger.debug("Failed to parse XML: %s", error) + return None + + def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = True) -> tuple[Iterator[str], int]: + """ + Parse the passed pom and extract the passed tags. + + Parameters + ---------- + pom : Element + The parsed POM. + tags : list[str] + The list of tags to try extracting from the POM. + resolve_properties: bool + Whether to attempt resolution of Maven properties within the POM. + + Returns + ------- + tuple[Iterator[str], int] : + The extracted contents of any matches tags, and the number of matches, as a tuple. + """ + results = [] + + # Try to match each tag with the contents of the POM. + for tag in tags: + element: typing.Optional[Element] = pom + + if tag.startswith("properties."): + # Tags under properties are often "." separated + # These can be safely split into two resulting tags as nested tags are not allowed here + tag_parts = ["properties", tag[11:]] + else: + # Other tags can be split into distinct elements via "." + tag_parts = tag.split(".") + + for index, tag_part in enumerate(tag_parts): + element = self._find_element(element, tag_part) + if element is None: + break + if index == len(tag_parts) - 1 and element.text: + # Add the contents of the final tag + results.append(element.text.strip()) + + # Resolve any Maven properties within the results + if resolve_properties: + results = self._resolve_properties(pom, results) + + return iter(results), len(results) + + def find_parent(self, pom: Element) -> tuple[str, str, str]: + """ + Extract parent information from passed POM. + + Parameters + ---------- + pom : str + The POM as a string. + + Returns + ------- + tuple[str] : + The GAV of the parent artifact. + """ + element = self._find_element(pom, "parent") + if element is None: + return "", "", "" + group = self._find_element(element, "groupId") + artifact = self._find_element(element, "artifactId") + version = self._find_element(element, "version") + if ( + group is not None + and group.text + and artifact is not None + and artifact.text + and version is not None + and version.text + ): + return group.text.strip(), artifact.text.strip(), version.text.strip() + return "", "", "" + + def _find_element(self, parent: typing.Optional[Element], target: str) -> typing.Optional[Element]: + if not parent: + return None + + # Attempt to match the target tag within the children of parent. + for child in parent: + # Handle raw tags, and tags accompanied by Maven metadata enclosed in curly braces. E.g. '{metadata}tag' + if child.tag == target or child.tag.endswith(f"}}{target}"): + return child + return None + + def _resolve_properties(self, pom: Element, values: list[str]) -> list[str]: + """Resolve any Maven properties found within the passed list of values. + + Maven POM files have five different use cases for properties (see https://maven.apache.org/pom.html). + Only the two that relate to contents found elsewhere within the same POM file are considered here. + That is: ${project.x} where x can be a child tag at any depth, or ${x} where x is found at project.properties.x. + Entries with unresolved properties are not included in the returned list. In the case of chained properties, + only the top most property is evaluated. + """ + resolved_values = [] + for value in values: + replacements: list = [] + # Calculate replacements - matches any number of ${...} entries in the current value + for match in re.finditer("\\$\\{[^}]+}", value): + text = match.group().replace("$", "").replace("{", "").replace("}", "") + if text.startswith("project."): + text = text.replace("project.", "") + else: + text = f"properties.{text}" + # Call find_scm with property resolution flag set to False to prevent the possibility of endless looping + value_iterator, count = self.find_scm(pom, [text], False) + if count == 0: + break + replacements.append([match.start(), next(value_iterator), match.end()]) + + # Apply replacements in reverse order + # E.g. + # git@github.com:owner/project${javac.src.version}-${project.inceptionYear}.git + # -> + # git@github.com:owner/project${javac.src.version}-2023.git + # -> + # git@github.com:owner/project1.8-2023.git + for replacement in reversed(replacements): + value = f"{value[:replacement[0]]}{replacement[1]}{value[replacement[2]:]}" + + resolved_values.append(value) + + return resolved_values diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index b03b64c1a..aeb0cc257 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -35,9 +35,10 @@ from macaron.errors import CloneError, DuplicateError, InvalidPURLError, PURLNotFoundError, RepoCheckOutError from macaron.output_reporter.reporter import FileReporter from macaron.output_reporter.results import Record, Report, SCMStatus +from macaron.repo_finder.repo_finder import find_repo from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.build_tool import BUILD_TOOLS +from macaron.slsa_analyzer.build_tool import BUILD_TOOLS, Gradle, Maven, Pip, Poetry # To load all checks into the registry from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 @@ -167,6 +168,10 @@ def run(self, user_config: dict, sbom_path: str = "", skip_deps: bool = False) - else: deps_resolved = self.resolve_dependencies(main_record.context, sbom_path) + # Use repo finder to find more repositories to analyze. + if defaults.getboolean("repofinder", "find_repos"): + self._resolve_more_dependencies(deps_resolved, main_record.context, main_record.description) + # Merge the automatically resolved dependencies with the manual configuration. deps_config = DependencyAnalyzer.merge_configs(deps_config, deps_resolved) @@ -1047,6 +1052,52 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: return results + def _resolve_more_dependencies( + self, dependencies: dict[str, DependencyInfo], context: AnalyzeContext, description: str + ) -> None: + """Utilise the Repo Finder to resolve more dependencies to repositories.""" + purl = PackageURL.from_string(context.component.purl) + build_tools = context.dynamic_data["build_spec"]["tools"] + if not purl.type or not build_tools: + logger.debug("No PURL type found for: %s", purl) + else: + # Get actual PURL type based on current build tool + build_tool = build_tools[0] + if len(build_tools) > 1: + logger.debug( + "More than one build tool found for %s, assuming type based on class: %s", + description, + build_tool.__class__, + ) + updated_type = "" + + if isinstance(build_tool, Gradle): + # Currently not a supported PURL type + updated_type = "gradle" + elif isinstance(build_tool, Maven): + updated_type = "maven" + elif isinstance(build_tool, (Pip, Poetry)): + updated_type = "pypi" + else: + # A type that is not yet supported here + logger.debug("Unsupported PURL type: %s", build_tool.__class__) + + for key, item in dependencies.items(): + if item["available"] != SCMStatus.MISSING_SCM: + continue + dep_namespace, dep_name = key.split(":") + purl_string = f"pkg:{updated_type}/{dep_namespace}/{dep_name}" + if item["version"] != "unspecified": + purl_string = purl_string + "@" + item["version"] + urls = find_repo(purl_string) + item["url"] = DependencyAnalyzer.find_valid_url(urls) + if item["url"] == "": + logger.debug("Failed to find url for purl: %s", purl_string) + else: + # TODO decide how to handle possible duplicates here + item["available"] = SCMStatus.AVAILABLE + item["note"] = "" + class DuplicateCmpError(DuplicateError): """This class is used for duplicated software component errors.""" diff --git a/tests/dependency_analyzer/java_repo_finder/__init__.py b/tests/repo_finder/__init__.py similarity index 100% rename from tests/dependency_analyzer/java_repo_finder/__init__.py rename to tests/repo_finder/__init__.py diff --git a/tests/repo_finder/repo_finder_dd/__init__.py b/tests/repo_finder/repo_finder_dd/__init__.py new file mode 100644 index 000000000..19aeac023 --- /dev/null +++ b/tests/repo_finder/repo_finder_dd/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py b/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py new file mode 100644 index 000000000..0a198119d --- /dev/null +++ b/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +""" +This module tests the python dd repo finder. +""" +from macaron.repo_finder.repo_finder_dd import RepoFinderDD + + +def test_repo_finder_dd() -> None: + """Test the functions of the repo finder.""" + repo_finder = RepoFinderDD("pypi") + assert ( + repo_finder.create_type_specific_url("", "packageurl-python") + == "https://api.deps.dev/v3alpha/systems/pypi/packages/packageurl-python" + ) diff --git a/tests/repo_finder/repo_finder_java/__init__.py b/tests/repo_finder/repo_finder_java/__init__.py new file mode 100644 index 000000000..19aeac023 --- /dev/null +++ b/tests/repo_finder/repo_finder_java/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/dependency_analyzer/java_repo_finder/resources/example_pom.xml b/tests/repo_finder/repo_finder_java/resources/example_pom.xml similarity index 100% rename from tests/dependency_analyzer/java_repo_finder/resources/example_pom.xml rename to tests/repo_finder/repo_finder_java/resources/example_pom.xml diff --git a/tests/dependency_analyzer/java_repo_finder/resources/example_pom_no_scm.xml b/tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml similarity index 100% rename from tests/dependency_analyzer/java_repo_finder/resources/example_pom_no_scm.xml rename to tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml diff --git a/tests/dependency_analyzer/java_repo_finder/test_java_repo_finder.py b/tests/repo_finder/repo_finder_java/test_repo_finder_java.py similarity index 74% rename from tests/dependency_analyzer/java_repo_finder/test_java_repo_finder.py rename to tests/repo_finder/repo_finder_java/test_repo_finder_java.py index 7783a3a3a..661c717d0 100644 --- a/tests/dependency_analyzer/java_repo_finder/test_java_repo_finder.py +++ b/tests/repo_finder/repo_finder_java/test_repo_finder_java.py @@ -7,27 +7,24 @@ import os from pathlib import Path -from macaron.config.defaults import defaults -from macaron.dependency_analyzer.java_repo_finder import create_urls, find_parent, find_scm, parse_pom +from macaron.repo_finder.repo_finder_java import JavaRepoFinder def test_java_repo_finder() -> None: """Test the functions of the repo finder.""" - repositories = defaults.get_list( - "repofinder.java", "artifact_repositories", fallback=["https://repo.maven.apache.org/maven2"] - ) group = "group" artifact = "artifact" version = "version" - created_urls = create_urls(group, artifact, version, repositories) + repo_finder = JavaRepoFinder() + created_urls = repo_finder.create_urls(group, artifact, version) assert created_urls resources_dir = Path(__file__).parent.joinpath("resources") with open(os.path.join(resources_dir, "example_pom.xml"), encoding="utf8") as file: file_data = file.read() - pom = parse_pom(file_data) + pom = repo_finder.parse_pom(file_data) assert pom is not None - found_urls, count = find_scm( + found_urls, count = repo_finder.find_scm( pom, ["scm.url", "scm.connection", "scm.developerConnection", "licenses.license.distribution"] ) assert count == 4 @@ -43,11 +40,12 @@ def test_java_repo_finder() -> None: def test_java_repo_finder_hierarchical() -> None: """Test the hierarchical capabilities of the repo finder.""" resources_dir = Path(__file__).parent.joinpath("resources") + repo_finder = JavaRepoFinder() with open(os.path.join(resources_dir, "example_pom_no_scm.xml"), encoding="utf8") as file: file_data = file.read() - pom = parse_pom(file_data) + pom = repo_finder.parse_pom(file_data) assert pom is not None - group, artifact, version = find_parent(pom) + group, artifact, version = repo_finder.find_parent(pom) assert group == "owner" assert artifact == "parent" assert version == "1" From 39c1279c78809ed347d646184c39725242ef3f02 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 28 Jul 2023 12:01:34 +1000 Subject: [PATCH 02/31] chore: enabled all supported languages in repo finder Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index f65ccb1d7..84089a2d2 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -40,7 +40,7 @@ def find_repo(purl_string: str) -> Iterator[str]: match purl.type: case "maven": repo_finder = JavaRepoFinder() - case "pypi": + case "pypi" | "nuget" | "cargo" | "npm": repo_finder = RepoFinderDD(purl.type) case _: From ede5b5843cfe447eadf1f0f29e52f8fa95850471 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 28 Jul 2023 12:16:29 +1000 Subject: [PATCH 03/31] chore: add configuration option for deps.dev and update docs Signed-off-by: Ben Selwyn-Smith --- docs/source/pages/using.rst | 6 +++++- src/macaron/config/defaults.ini | 1 + src/macaron/repo_finder/repo_finder.py | 22 ++++++++++++---------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/source/pages/using.rst b/docs/source/pages/using.rst index b424d1a83..95a0e2352 100644 --- a/docs/source/pages/using.rst +++ b/docs/source/pages/using.rst @@ -209,18 +209,22 @@ Within the configuration file under the ``repofinder.java`` header, three option - ``repo_pom_paths`` (Values: List of POM tags) - Determines where to search for repository information in the POM files. E.g. scm.url. - ``find_parents`` (Values: True or False) - When enabled, the Repository Finding feature will also search for repository URLs in parents POM files of the current dependency. -The entire feature can be disabled via the ``find_repos`` option found under the configuration header ``repofinder``, as so: +Under the related header ``repofinder``, two more options exist: ``find_repos``, and ``use_open_source_insights``: - ``find_repos`` (Values: True or False) - Enables or disables the Repository Finding feature. +- ``use_open_source_insights`` (Values: True or False) - Enables or disables use of Google's Open Source Insights API. .. note:: Finding repositories requires at least one remote call, adding some additional overhead to an analysis run. +.. note:: Google's Open Source Insights API is currently used to find repositories for: Python, Rust, .Net, NodeJS + An example configuration file for utilising this feature: .. code-block:: ini [repofinder] find_repos = True + use_open_source_insights = True [repofinder.java] artifact_repositories = https://repo.maven.apache.org/maven2 diff --git a/src/macaron/config/defaults.ini b/src/macaron/config/defaults.ini index fc67d129c..abca6969e 100644 --- a/src/macaron/config/defaults.ini +++ b/src/macaron/config/defaults.ini @@ -46,6 +46,7 @@ recursive = False # This is the repo finder script. [repofinder] find_repos = True +use_open_source_insights = True [repofinder.java] # The list of maven-like repositories to attempt to retrieve artifact POMs from. diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 84089a2d2..6b7d84b19 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -8,6 +8,7 @@ from packageurl import PackageURL +from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_dd import RepoFinderDD from macaron.repo_finder.repo_finder_java import JavaRepoFinder @@ -35,17 +36,18 @@ def find_repo(purl_string: str) -> Iterator[str]: logger.debug("Invalid PURL: %s, %s", purl_string, error) return + # Find matching Repo Finder repo_finder: BaseRepoFinder + if purl.type == "maven": + repo_finder = JavaRepoFinder() + elif purl.type in ["pypi", "nuget", "cargo", "npm"] and defaults.getboolean( + "repofinder", "use_open_source_insights" + ): + repo_finder = RepoFinderDD(purl.type) + else: + logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl_string) + return - match purl.type: - case "maven": - repo_finder = JavaRepoFinder() - case "pypi" | "nuget" | "cargo" | "npm": - repo_finder = RepoFinderDD(purl.type) - - case _: - logger.debug("Unsupported type in PURL: %s (%s)", purl.type, purl_string) - return - + # Call Repo Finder logger.debug("Analyzing %s with Repo Finder: %s", purl_string, repo_finder.__class__) yield from repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") From 1a171ff440d96f4bd5f3ba5cb5435238670ed694 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 1 Aug 2023 16:24:43 +1000 Subject: [PATCH 04/31] chore: addressed PR feedback Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder_base.py | 6 ++-- src/macaron/repo_finder/repo_finder_dd.py | 20 ++++++------ src/macaron/repo_finder/repo_finder_java.py | 32 +++++++++---------- .../repo_finder_java/test_repo_finder_java.py | 6 ++-- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py index 100694113..f48419d31 100644 --- a/src/macaron/repo_finder/repo_finder_base.py +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -71,7 +71,7 @@ def retrieve_metadata(self, session: requests.Session, url: str) -> str: """ @abstractmethod - def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + def read_metadata(self, metadata: str) -> list[str]: """ Parse the passed metadata and extract the relevant information. @@ -82,6 +82,6 @@ def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: Returns ------- - tuple[Iterator[str], int] : - The extracted contents in iterable form and count as a tuple. + list[str] : + The extracted contents as a list of strings. """ diff --git a/src/macaron/repo_finder/repo_finder_dd.py b/src/macaron/repo_finder/repo_finder_dd.py index 254309d5c..496ab4a4b 100644 --- a/src/macaron/repo_finder/repo_finder_dd.py +++ b/src/macaron/repo_finder/repo_finder_dd.py @@ -15,7 +15,7 @@ class RepoFinderDD(BaseRepoFinder): - """This class is used to find repositories using Google's Open Source Insights tool (deps.dev).""" + """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev (DD).""" # The label used by deps.dev to denote repository urls (Based on observation ONLY) repo_url_label = "SOURCE_REPO" @@ -59,12 +59,12 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: logger.debug("Failed to retrieve metadata for: %s", artifact) return - urls, url_count = self.read_metadata(metadata) - if url_count == 0: + urls = self.read_metadata(metadata) + if not urls: logger.debug("Failed to extract repository URLs from metadata: %s", artifact) return - yield from urls + yield from iter(urls) def create_urls(self, group: str, artifact: str, version: str) -> list[str]: """ @@ -141,7 +141,7 @@ def retrieve_metadata(self, session: requests.Session, url: str) -> str: return result.text - def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + def read_metadata(self, metadata: str) -> list[str]: """ Parse the deps.dev metadata and extract the repository links. @@ -152,20 +152,20 @@ def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: Returns ------- - tuple[Iterator[str], int] : - The extracted contents in iterable form and count as a tuple. + list[str] : + The extracted contents as a list of strings. """ parsed = json.loads(metadata) if not parsed["links"]: logger.debug("Metadata had no URLs: %s", parsed["versionKey"]) - return iter([]), 0 + return [] for link in parsed["links"]: if link["label"] == self.repo_url_label: - return iter([link["url"]]), 1 + return list(link["url"]) - return iter([]), 0 + return [] def create_type_specific_url(self, namespace: str, name: str) -> str: """Create a url for the deps.dev API based on the package type. diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index c5ac090aa..2a7afb202 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -73,11 +73,11 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: logger.debug("No POM found for %s:%s:%s", group, artifact, version) return - urls, url_count = self.read_metadata(pom) + urls = self.read_metadata(pom) - if url_count > 0: - logger.debug("Found %s urls.", url_count) - yield from urls + if urls: + logger.debug("Found %s urls.", len(urls)) + yield from iter(urls) if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: # Attempt to extract parent information from POM @@ -146,7 +146,7 @@ def retrieve_metadata(self, session: requests.Session, url: str) -> str: return res.text - def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: + def read_metadata(self, metadata: str) -> list[str]: """ Parse the passed pom and extract the relevant tags. @@ -157,19 +157,19 @@ def read_metadata(self, metadata: str) -> tuple[Iterator[str], int]: Returns ------- - tuple[Iterator[str], int] : - The extracted contents in iterable form and count as a tuple. + list[str] : + The extracted contents as a list of strings. """ # Retrieve tags tags = defaults.get_list("repofinder.java", "repo_pom_paths") if not any(tags): logger.debug("No POM tags found for URL discovery.") - return iter([]), 0 + return [] # Parse POM using defusedxml pom_element = self.parse_pom(metadata) if pom_element is None: - return iter([]), 0 + return [] # Attempt to extract SCM data and return URL return self.find_scm(pom_element, tags) @@ -195,7 +195,7 @@ def parse_pom(self, pom: str) -> Element | None: logger.debug("Failed to parse XML: %s", error) return None - def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = True) -> tuple[Iterator[str], int]: + def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = True) -> list[str]: """ Parse the passed pom and extract the passed tags. @@ -210,8 +210,8 @@ def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = Tru Returns ------- - tuple[Iterator[str], int] : - The extracted contents of any matches tags, and the number of matches, as a tuple. + list[str] : + The extracted contents of any matches tags as a list of strings. """ results = [] @@ -239,7 +239,7 @@ def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = Tru if resolve_properties: results = self._resolve_properties(pom, results) - return iter(results), len(results) + return results def find_parent(self, pom: Element) -> tuple[str, str, str]: """ @@ -303,10 +303,10 @@ def _resolve_properties(self, pom: Element, values: list[str]) -> list[str]: else: text = f"properties.{text}" # Call find_scm with property resolution flag set to False to prevent the possibility of endless looping - value_iterator, count = self.find_scm(pom, [text], False) - if count == 0: + result = self.find_scm(pom, [text], False) + if not result: break - replacements.append([match.start(), next(value_iterator), match.end()]) + replacements.append([match.start(), result[0], match.end()]) # Apply replacements in reverse order # E.g. diff --git a/tests/repo_finder/repo_finder_java/test_repo_finder_java.py b/tests/repo_finder/repo_finder_java/test_repo_finder_java.py index 661c717d0..e57e35057 100644 --- a/tests/repo_finder/repo_finder_java/test_repo_finder_java.py +++ b/tests/repo_finder/repo_finder_java/test_repo_finder_java.py @@ -24,17 +24,17 @@ def test_java_repo_finder() -> None: file_data = file.read() pom = repo_finder.parse_pom(file_data) assert pom is not None - found_urls, count = repo_finder.find_scm( + found_urls = repo_finder.find_scm( pom, ["scm.url", "scm.connection", "scm.developerConnection", "licenses.license.distribution"] ) - assert count == 4 + assert len(found_urls) == 4 expected = [ "https://github.com/owner/project", "ssh://git@hostname:port/owner/Example_License.git", "git@github.com:owner/project1.8-2023.git", "${licenses.license.distribution}", ] - assert expected == list(found_urls) + assert expected == found_urls def test_java_repo_finder_hierarchical() -> None: From 6ab78f5f01b01af271db4477ced28fd1d0422009 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 3 Aug 2023 10:31:01 +1000 Subject: [PATCH 05/31] chore: add integration test for repo finder Signed-off-by: Ben Selwyn-Smith --- scripts/dev_scripts/integration_tests.sh | 11 ++++++ .../dev_scripts/integration_tests_docker.sh | 11 ++++++ src/macaron/__main__.py | 34 +++++++++++++++++++ src/macaron/repo_finder/repo_finder_dd.py | 2 +- 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 78cd2689c..5de8b0b56 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -532,3 +532,14 @@ then echo -e "Expected zero status code but got $RESULT_CODE." exit 1 fi + +# Testing the Repo Finder's remote calls. +echo -e "\n----------------------------------------------------------------------------------" +echo "Verify Repo Finder functionality." +echo -e "----------------------------------------------------------------------------------\n" +macaron -v verify-repo-finder +if [ $? -ne 0 ]; +then + echo -e "Expect zero status code but got $?." + log_fail +fi diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 20ecd3327..79e55e249 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -145,6 +145,17 @@ POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/policy_report.js $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail +# Testing the Repo Finder's remote calls. +echo -e "\n----------------------------------------------------------------------------------" +echo "Verify Repo Finder functionality." +echo -e "----------------------------------------------------------------------------------\n" +$RUN_MACARON_SCRIPT -v verify-repo-finder +if [ $? -ne 0 ]; +then + echo -e "Expect zero status code but got $?." + log_fail +fi + echo -e "\n----------------------------------------------------------------------------------" echo "Test running the analysis without setting the GITHUB_TOKEN environment variables." echo -e "----------------------------------------------------------------------------------\n" diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index d9b83e268..f9939fd4a 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -9,6 +9,7 @@ import sys from importlib import metadata as importlib_metadata +import requests from jinja2 import Environment, FileSystemLoader, select_autoescape import macaron @@ -18,6 +19,7 @@ from macaron.output_reporter.reporter import HTMLReporter, JSONReporter, PolicyReporter from macaron.parsers.yaml.loader import YamlLoader from macaron.policy_engine.policy_engine import run_policy_engine, show_prelude +from macaron.repo_finder.repo_finder_dd import RepoFinderDD from macaron.slsa_analyzer.analyzer import Analyzer from macaron.slsa_analyzer.git_service import GIT_SERVICES from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES @@ -158,6 +160,32 @@ def verify_policy(verify_policy_args: argparse.Namespace) -> int: return os.EX_USAGE +def verify_repo_finder() -> int: + """Verify the functionality of the remote API calls used by the repo finder. + + Functionality relating to Java artifacts is not verified for two reasons: + - It is extremely unlikely that Maven central will change its API or cease operation in the near future. + - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. + """ + # Verify deps.dev for a Python package + repo_finder = RepoFinderDD("pypi") + urls = [] + # Without version + urls.append(repo_finder.create_urls("", "packageurl-python", "")) + # With version + urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) + with requests.Session() as session: + for url in urls: + logger.debug("Verifying: %s", url[0]) + metadata = repo_finder.retrieve_metadata(session, url[0]) + if not metadata: + return 1 + links = repo_finder.read_metadata(metadata) + if not links: + return 1 + return 0 + + def perform_action(action_args: argparse.Namespace) -> None: """Perform the indicated action of Macaron.""" match action_args.action: @@ -169,6 +197,9 @@ def perform_action(action_args: argparse.Namespace) -> None: case "verify-policy": sys.exit(verify_policy(action_args)) + case "verify-repo-finder": + sys.exit(verify_repo_finder()) + case "analyze": # Check that the GitHub token is enabled. gh_token = os.environ.get("GITHUB_TOKEN") @@ -347,6 +378,9 @@ def main(argv: list[str] | None = None) -> None: vp_parser = sub_parser.add_parser(name="verify-policy") vp_group = vp_parser.add_mutually_exclusive_group(required=True) + # Verify the Repo Finder + sub_parser.add_parser(name="verify-repo-finder") + vp_parser.add_argument("-d", "--database", required=True, type=str, help="Path to the database.") vp_group.add_argument("-f", "--file", type=str, help="Path to the Datalog policy.") vp_group.add_argument("-s", "--show-prelude", action="store_true", help="Show policy prelude.") diff --git a/src/macaron/repo_finder/repo_finder_dd.py b/src/macaron/repo_finder/repo_finder_dd.py index 496ab4a4b..1472ff4a1 100644 --- a/src/macaron/repo_finder/repo_finder_dd.py +++ b/src/macaron/repo_finder/repo_finder_dd.py @@ -86,7 +86,7 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: list[str] The list of created URLs. """ - package_name = f"{group}:{artifact}:{version}" + package_name = f"{group}:{artifact}@{version}" base_url = self.create_type_specific_url(group, artifact) if version: From 36f22c5b12705d191d6244ee9dcf3c9a71d4d879 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 8 Aug 2023 09:40:42 +1000 Subject: [PATCH 06/31] chore: addressed review feedback Signed-off-by: Ben Selwyn-Smith --- src/macaron/__main__.py | 22 ++++---- src/macaron/config/global_config.py | 1 - src/macaron/repo_finder/repo_finder.py | 4 +- src/macaron/repo_finder/repo_finder_base.py | 8 +-- ...o_finder_dd.py => repo_finder_deps_dev.py} | 50 ++++++------------- src/macaron/repo_finder/repo_finder_java.py | 32 ++++-------- .../repo_finder_dd/test_repo_finder_dd.py | 4 +- 7 files changed, 43 insertions(+), 78 deletions(-) rename src/macaron/repo_finder/{repo_finder_dd.py => repo_finder_deps_dev.py} (76%) diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index f9939fd4a..675e3e1f8 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -9,7 +9,6 @@ import sys from importlib import metadata as importlib_metadata -import requests from jinja2 import Environment, FileSystemLoader, select_autoescape import macaron @@ -19,7 +18,7 @@ from macaron.output_reporter.reporter import HTMLReporter, JSONReporter, PolicyReporter from macaron.parsers.yaml.loader import YamlLoader from macaron.policy_engine.policy_engine import run_policy_engine, show_prelude -from macaron.repo_finder.repo_finder_dd import RepoFinderDD +from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev from macaron.slsa_analyzer.analyzer import Analyzer from macaron.slsa_analyzer.git_service import GIT_SERVICES from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES @@ -168,21 +167,20 @@ def verify_repo_finder() -> int: - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. """ # Verify deps.dev for a Python package - repo_finder = RepoFinderDD("pypi") + repo_finder = RepoFinderDepsDev("pypi") urls = [] # Without version urls.append(repo_finder.create_urls("", "packageurl-python", "")) # With version urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) - with requests.Session() as session: - for url in urls: - logger.debug("Verifying: %s", url[0]) - metadata = repo_finder.retrieve_metadata(session, url[0]) - if not metadata: - return 1 - links = repo_finder.read_metadata(metadata) - if not links: - return 1 + for url in urls: + logger.debug("Verifying: %s", url[0]) + metadata = repo_finder.retrieve_metadata(url[0]) + if not metadata: + return 1 + links = repo_finder.read_metadata(metadata) + if not links: + return 1 return 0 diff --git a/src/macaron/config/global_config.py b/src/macaron/config/global_config.py index 82a187c19..c2b1076f1 100644 --- a/src/macaron/config/global_config.py +++ b/src/macaron/config/global_config.py @@ -21,7 +21,6 @@ class GlobalConfig: gh_token: str = "" debug_level: int = logging.DEBUG resources_path: str = "" - find_repos: bool = True def load( self, diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 6b7d84b19..45f7cbecb 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -10,7 +10,7 @@ from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.repo_finder.repo_finder_dd import RepoFinderDD +from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev from macaron.repo_finder.repo_finder_java import JavaRepoFinder logger: logging.Logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ def find_repo(purl_string: str) -> Iterator[str]: elif purl.type in ["pypi", "nuget", "cargo", "npm"] and defaults.getboolean( "repofinder", "use_open_source_insights" ): - repo_finder = RepoFinderDD(purl.type) + repo_finder = RepoFinderDepsDev(purl.type) else: logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl_string) return diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py index f48419d31..ee496f53c 100644 --- a/src/macaron/repo_finder/repo_finder_base.py +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -6,8 +6,6 @@ from abc import ABC, abstractmethod from collections.abc import Iterator -import requests - class BaseRepoFinder(ABC): """This abstract class is used to represent Repository Finders.""" @@ -53,14 +51,12 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: """ @abstractmethod - def retrieve_metadata(self, session: requests.Session, url: str) -> str: + def retrieve_metadata(self, url: str) -> str: """ - Attempt to retrieve the file located at the passed URL using the passed Session. + Attempt to retrieve the file located at the passed URL. Parameters ---------- - session : requests.Session - The HTTP session to use for attempting the GET request. url : str The URL for the GET request. diff --git a/src/macaron/repo_finder/repo_finder_dd.py b/src/macaron/repo_finder/repo_finder_deps_dev.py similarity index 76% rename from src/macaron/repo_finder/repo_finder_dd.py rename to src/macaron/repo_finder/repo_finder_deps_dev.py index 1472ff4a1..a4df40545 100644 --- a/src/macaron/repo_finder/repo_finder_dd.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -7,14 +7,13 @@ from collections.abc import Iterator from urllib.parse import quote as encode -import requests - from macaron.repo_finder.repo_finder_base import BaseRepoFinder +from macaron.util import send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) -class RepoFinderDD(BaseRepoFinder): +class RepoFinderDepsDev(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev (DD).""" # The label used by deps.dev to denote repository urls (Based on observation ONLY) @@ -53,11 +52,10 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: logger.debug("No urls found for: %s", artifact) return - with requests.Session() as session: - metadata = self.retrieve_metadata(session, request_urls[0]) - if not metadata: - logger.debug("Failed to retrieve metadata for: %s", artifact) - return + metadata = self.retrieve_metadata(request_urls[0]) + if not metadata: + logger.debug("Failed to retrieve metadata for: %s", artifact) + return urls = self.read_metadata(metadata) if not urls: @@ -86,25 +84,17 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: list[str] The list of created URLs. """ - package_name = f"{group}:{artifact}@{version}" base_url = self.create_type_specific_url(group, artifact) if version: return [f"{base_url}/versions/{version}"] # Find the latest version. - with requests.Session() as session: - try: - result = session.get(base_url) - except (requests.RequestException, OSError) as error: - logger.debug("Error during version retrieval: %s", error) - return [] - - if not result.ok: - logger.debug("Failed to retrieve versions for: %s, error code: %s", package_name, result.status_code) - return [] - - metadata = json.loads(result.text) + response = send_get_http_raw(base_url, {}) + if not response: + return [] + + metadata = json.loads(response.text) versions = metadata["versions"] latest_version = versions[len(version) - 1]["versionKey"]["version"] @@ -113,14 +103,12 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: return [] - def retrieve_metadata(self, session: requests.Session, url: str) -> str: + def retrieve_metadata(self, url: str) -> str: """ - Attempt to retrieve the file located at the passed URL using the passed Session. + Attempt to retrieve the file located at the passed URL. Parameters ---------- - session : requests.Session - The HTTP session to use for attempting the GET request. url : str The URL for the GET request. @@ -129,17 +117,11 @@ def retrieve_metadata(self, session: requests.Session, url: str) -> str: str : The retrieved file data or an empty string. """ - try: - result = session.get(url) - except (requests.RequestException, OSError) as error: - logger.debug("Error during metadata retrieval: %s", error) - return "" - - if not result.ok: - logger.debug("Failed to retrieve metadata at: %s, error code: %s", url, result.status_code) + response = send_get_http_raw(url, {}) + if not response: return "" - return result.text + return response.text def read_metadata(self, metadata: str) -> list[str]: """ diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 2a7afb202..addc21e44 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -9,11 +9,11 @@ from xml.etree.ElementTree import Element # nosec import defusedxml.ElementTree -import requests from defusedxml.ElementTree import fromstring from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder +from macaron.util import send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -61,12 +61,11 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: return # Try each POM URL in order, terminating early if a match is found - with requests.Session() as session: - pom = "" - for request_url in request_urls: - pom = self.retrieve_metadata(session, request_url) - if pom != "": - break + pom = "" + for request_url in request_urls: + pom = self.retrieve_metadata(request_url) + if pom != "": + break if pom == "": # Abort if no POM was found @@ -116,14 +115,12 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: urls.append(f"{repo}/{group}/{artifact}/{version}/{artifact}-{version}.pom") return urls - def retrieve_metadata(self, session: requests.Session, url: str) -> str: + def retrieve_metadata(self, url: str) -> str: """ - Attempt to retrieve the file located at the passed URL using the passed Session. + Attempt to retrieve the file located at the passed URL. Parameters ---------- - session : requests.Session - The HTTP session to use for attempting the GET request. url : str The URL for the GET request. @@ -132,19 +129,12 @@ def retrieve_metadata(self, session: requests.Session, url: str) -> str: str : The retrieved file data or an empty string. """ - try: - res = session.get(url) - except (requests.RequestException, OSError) as error: - logger.debug("Error during pom retrieval: %s", error) - return "" - - if not res.ok: - logger.debug("Failed to retrieve pom from: %s, error code: %s", url, res.status_code) + response = send_get_http_raw(url, {}) + if not response: return "" logger.debug("Found artifact POM at: %s", url) - - return res.text + return response.text def read_metadata(self, metadata: str) -> list[str]: """ diff --git a/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py b/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py index 0a198119d..0c945d5c8 100644 --- a/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py +++ b/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py @@ -4,12 +4,12 @@ """ This module tests the python dd repo finder. """ -from macaron.repo_finder.repo_finder_dd import RepoFinderDD +from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev def test_repo_finder_dd() -> None: """Test the functions of the repo finder.""" - repo_finder = RepoFinderDD("pypi") + repo_finder = RepoFinderDepsDev("pypi") assert ( repo_finder.create_type_specific_url("", "packageurl-python") == "https://api.deps.dev/v3alpha/systems/pypi/packages/packageurl-python" From 7e725c2194748649d3bb1bcbac84be7cec2dbf04 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 29 Aug 2023 14:47:53 +1000 Subject: [PATCH 07/31] chore: addressed PR feedback; rebased and refactored Signed-off-by: Ben Selwyn-Smith --- scripts/dev_scripts/integration_tests.sh | 4 +- .../dev_scripts/integration_tests_docker.sh | 4 +- src/macaron/__main__.py | 18 +- src/macaron/dependency_analyzer/cyclonedx.py | 4 +- .../dependency_resolver.py | 2 +- src/macaron/repo_finder/repo_finder.py | 118 ++++++++++--- .../repo_finder/repo_finder_deps_dev.py | 31 ++-- src/macaron/repo_finder/repo_finder_java.py | 13 +- src/macaron/slsa_analyzer/analyzer.py | 158 ++++-------------- src/macaron/slsa_analyzer/git_url.py | 1 + .../test_dependency_analyzer.py | 12 +- .../expected_results/policy_report.json | 2 +- tests/slsa_analyzer/test_analyzer.py | 43 ----- 13 files changed, 185 insertions(+), 225 deletions(-) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 5de8b0b56..d43d45c1e 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -535,9 +535,9 @@ fi # Testing the Repo Finder's remote calls. echo -e "\n----------------------------------------------------------------------------------" -echo "Verify Repo Finder functionality." +echo "Testing Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -macaron -v verify-repo-finder +macaron -v test-repo-finder if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 79e55e249..bb78d1431 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -147,9 +147,9 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail # Testing the Repo Finder's remote calls. echo -e "\n----------------------------------------------------------------------------------" -echo "Verify Repo Finder functionality." +echo "Test Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -$RUN_MACARON_SCRIPT -v verify-repo-finder +$RUN_MACARON_SCRIPT -v test-repo-finder if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index 675e3e1f8..0a813d2a8 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -159,14 +159,14 @@ def verify_policy(verify_policy_args: argparse.Namespace) -> int: return os.EX_USAGE -def verify_repo_finder() -> int: - """Verify the functionality of the remote API calls used by the repo finder. +def test_repo_finder() -> int: + """Test the functionality of the remote API calls used by the repo finder. Functionality relating to Java artifacts is not verified for two reasons: - It is extremely unlikely that Maven central will change its API or cease operation in the near future. - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. """ - # Verify deps.dev for a Python package + # Test deps.dev API for a Python package repo_finder = RepoFinderDepsDev("pypi") urls = [] # Without version @@ -174,14 +174,14 @@ def verify_repo_finder() -> int: # With version urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) for url in urls: - logger.debug("Verifying: %s", url[0]) + logger.debug("Testing: %s", url[0]) metadata = repo_finder.retrieve_metadata(url[0]) if not metadata: - return 1 + return os.EX_UNAVAILABLE links = repo_finder.read_metadata(metadata) if not links: - return 1 - return 0 + return os.EX_UNAVAILABLE + return os.EX_OK def perform_action(action_args: argparse.Namespace) -> None: @@ -195,8 +195,8 @@ def perform_action(action_args: argparse.Namespace) -> None: case "verify-policy": sys.exit(verify_policy(action_args)) - case "verify-repo-finder": - sys.exit(verify_repo_finder()) + case "test-repo-finder": + sys.exit(test_repo_finder()) case "analyze": # Check that the GitHub token is enabled. diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 61d2e924f..6029b058e 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -172,7 +172,7 @@ def convert_components_to_artifacts( # See https://peps.python.org/pep-0589/#totality item = DependencyInfo( version=component.get("version") or "", - group=component.get("group") or "", + namespace=component.get("group") or "", name=component.get("name") or "", purl=component.get("purl") or "", url="", @@ -190,7 +190,7 @@ def convert_components_to_artifacts( "snapshot" in (item.get("version") or "").lower() # or "" is not necessary but mypy produces a FP otherwise. and root_component - and item.get("group") == root_component.get("group") + and item.get("namespace") == root_component.get("group") ): continue logger.debug( diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index 6de1d0888..c76b64966 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -30,7 +30,7 @@ class DependencyInfo(TypedDict): """The information of a resolved dependency.""" version: str - group: str + namespace: str name: str purl: str url: str diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 45f7cbecb..3904538c1 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -4,11 +4,13 @@ """This module contains the logic for using/calling the different repo finders.""" import logging -from collections.abc import Iterator +import os +from urllib.parse import ParseResult, urlunparse from packageurl import PackageURL from macaron.config.defaults import defaults +from macaron.dependency_analyzer import DependencyAnalyzer from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev from macaron.repo_finder.repo_finder_java import JavaRepoFinder @@ -16,38 +18,106 @@ logger: logging.Logger = logging.getLogger(__name__) -def find_repo(purl_string: str) -> Iterator[str]: +def find_repo(purl: PackageURL) -> str: """Retrieve the repository URL that matches the given PURL. Parameters ---------- - purl_string : str - The purl string representing a package. + purl : PackageURL + The parsed PURL to convert to the repository path. - Yields - ------ - Iterator[str] : - The repository URLs found for the passed package. + Returns + ------- + str : + The repository URL found for the passed package. """ - # Parse the purl string - try: - purl = PackageURL.from_string(purl_string) - except ValueError as error: - logger.debug("Invalid PURL: %s, %s", purl_string, error) - return - - # Find matching Repo Finder repo_finder: BaseRepoFinder if purl.type == "maven": repo_finder = JavaRepoFinder() - elif purl.type in ["pypi", "nuget", "cargo", "npm"] and defaults.getboolean( - "repofinder", "use_open_source_insights" - ): + elif defaults.getboolean("repofinder", "use_open_source_insights") and purl.type in [ + "pypi", + "nuget", + "cargo", + "npm", + ]: repo_finder = RepoFinderDepsDev(purl.type) else: - logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl_string) - return + logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl.to_string()) + return "" - # Call Repo Finder - logger.debug("Analyzing %s with Repo Finder: %s", purl_string, repo_finder.__class__) - yield from repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") + # Call Repo Finder and return first valid URL + logger.debug("Analyzing %s with Repo Finder: %s", purl.to_string(), repo_finder.__class__) + found_urls = repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") + return DependencyAnalyzer.find_valid_url(found_urls) + + +def to_domain_from_known_purl_types(purl_type: str) -> str | None: + """Return the git service domain from a known web-based purl type. + + This method is used to handle cases where the purl type value is not the git domain but a pre-defined + repo-based type in https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst. + + Note that this method will be updated when there are new pre-defined types as per the PURL specification. + + Parameters + ---------- + purl_type : str + The type field of the PURL. + + Returns + ------- + str | None + The git service domain corresponding to the purl type or None if the purl type is unknown. + """ + known_types = {"github": "github.com", "bitbucket": "bitbucket.org"} + return known_types.get(purl_type, None) + + +def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: + """Return the repository path from the PURL string. + + This method only supports converting a PURL with the following format: + + pkg://[...] + + Where ``type`` could be either: + - The pre-defined repository-based PURL type as defined in + https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst + + - The supprted git service domains (e.g. ``github.com``) defined in ``available_domains``. + + The repository path will be generated with the following format ``https:////``. + + Parameters + ---------- + purl : PackageURL + The parsed PURL to convert to the repository path. + available_domains: list[str] + The list of available domains + + Returns + ------- + str | None + The URL to the repository which the PURL is referring to or None if we cannot convert it. + """ + domain = to_domain_from_known_purl_types(purl.type) or (purl.type if purl.type in available_domains else None) + if not domain: + logger.error("The PURL type of %s is not valid as a repository type.", purl.to_string()) + # Try to find the repository + return find_repo(purl) + + if not purl.namespace: + logger.error("Expecting a non-empty namespace from %s.", purl.to_string()) + return None + + # TODO: Handle the version tag and commit digest if they are given in the PURL. + return urlunparse( + ParseResult( + scheme="https", + netloc=domain, + path=os.path.join(purl.namespace, purl.name), + params="", + query="", + fragment="", + ) + ) diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index a4df40545..4385f9af5 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -7,6 +7,8 @@ from collections.abc import Iterator from urllib.parse import quote as encode +from requests.exceptions import ReadTimeout + from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.util import send_get_http_raw @@ -16,9 +18,6 @@ class RepoFinderDepsDev(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev (DD).""" - # The label used by deps.dev to denote repository urls (Based on observation ONLY) - repo_url_label = "SOURCE_REPO" - def __init__(self, purl_type: str) -> None: """Initialise the deps.dev repository finder instance. @@ -86,11 +85,19 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: """ base_url = self.create_type_specific_url(group, artifact) + if not base_url: + return [] + if version: return [f"{base_url}/versions/{version}"] # Find the latest version. - response = send_get_http_raw(base_url, {}) + try: + response = send_get_http_raw(base_url, {}) + except ReadTimeout: + logger.debug("Failed to retrieve version (timeout): %s:%s", group, artifact) + return [] + if not response: return [] @@ -99,6 +106,7 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: latest_version = versions[len(version) - 1]["versionKey"]["version"] if latest_version: + logger.debug("Found latest version: %s", latest_version) return [f"{base_url}/versions/{latest_version}"] return [] @@ -117,7 +125,12 @@ def retrieve_metadata(self, url: str) -> str: str : The retrieved file data or an empty string. """ - response = send_get_http_raw(url, {}) + try: + response = send_get_http_raw(url, {}) + except ReadTimeout: + logger.debug("Failed to retrieve metadata (timeout): %s", url) + return "" + if not response: return "" @@ -143,11 +156,7 @@ def read_metadata(self, metadata: str) -> list[str]: logger.debug("Metadata had no URLs: %s", parsed["versionKey"]) return [] - for link in parsed["links"]: - if link["label"] == self.repo_url_label: - return list(link["url"]) - - return [] + return list(parsed["links"]) def create_type_specific_url(self, namespace: str, name: str) -> str: """Create a url for the deps.dev API based on the package type. @@ -177,6 +186,8 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: package_name = name case "nuget" | "cargo": package_name = name + case "maven": + package_name = f"{namespace}%3A{name}" case _: logger.debug("PURL type not yet supported: %s", self.type) diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index addc21e44..225729efb 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -4,12 +4,12 @@ """This module contains the JavaRepoFinder class to be used for finding Java repositories.""" import logging import re -import typing from collections.abc import Iterator from xml.etree.ElementTree import Element # nosec import defusedxml.ElementTree from defusedxml.ElementTree import fromstring +from requests.exceptions import ReadTimeout from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder @@ -129,7 +129,12 @@ def retrieve_metadata(self, url: str) -> str: str : The retrieved file data or an empty string. """ - response = send_get_http_raw(url, {}) + try: + response = send_get_http_raw(url, {}) + except ReadTimeout: + logger.debug("Failed to retrieve metadata (timeout): %s", url) + return "" + if not response: return "" @@ -207,7 +212,7 @@ def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = Tru # Try to match each tag with the contents of the POM. for tag in tags: - element: typing.Optional[Element] = pom + element: Element | None = pom if tag.startswith("properties."): # Tags under properties are often "." separated @@ -262,7 +267,7 @@ def find_parent(self, pom: Element) -> tuple[str, str, str]: return group.text.strip(), artifact.text.strip(), version.text.strip() return "", "", "" - def _find_element(self, parent: typing.Optional[Element], target: str) -> typing.Optional[Element]: + def _find_element(self, parent: Element | None, target: str) -> Element | None: if not parent: return None diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index aeb0cc257..07eda46fe 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -11,7 +11,6 @@ from datetime import datetime, timezone from pathlib import Path from typing import NamedTuple -from urllib.parse import ParseResult, urlunparse import sqlalchemy.exc from git import InvalidGitRepositoryError @@ -35,10 +34,11 @@ from macaron.errors import CloneError, DuplicateError, InvalidPURLError, PURLNotFoundError, RepoCheckOutError from macaron.output_reporter.reporter import FileReporter from macaron.output_reporter.results import Record, Report, SCMStatus +from macaron.repo_finder import repo_finder from macaron.repo_finder.repo_finder import find_repo from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.build_tool import BUILD_TOOLS, Gradle, Maven, Pip, Poetry +from macaron.slsa_analyzer.build_tool import BUILD_TOOLS # To load all checks into the registry from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 @@ -168,10 +168,6 @@ def run(self, user_config: dict, sbom_path: str = "", skip_deps: bool = False) - else: deps_resolved = self.resolve_dependencies(main_record.context, sbom_path) - # Use repo finder to find more repositories to analyze. - if defaults.getboolean("repofinder", "find_repos"): - self._resolve_more_dependencies(deps_resolved, main_record.context, main_record.description) - # Merge the automatically resolved dependencies with the manual configuration. deps_config = DependencyAnalyzer.merge_configs(deps_config, deps_resolved) @@ -343,11 +339,14 @@ def resolve_dependencies(self, main_ctx: AnalyzeContext, sbom_path: str) -> dict # We collect the generated SBOM as a best effort, even if the build exits with errors. # TODO: add improvements to help the SBOM build succeed as much as possible. - # Update deps_resolved with new dependencies. deps_resolved |= dep_analyzer.collect_dependencies(str(working_dir)) logger.info("Stored dependency resolver log for %s to %s.", dep_analyzer.tool_name, log_path) + # Use repo finder to find more repositories to analyze. + if defaults.getboolean("repofinder", "find_repos"): + self._resolve_more_dependencies(deps_resolved) + return deps_resolved def run_single( @@ -503,80 +502,6 @@ def add_repository(self, branch_name: str, git_obj: Git) -> Repository | None: return repository - @staticmethod - def to_domain_from_known_purl_types(purl_type: str) -> str | None: - """Return the git service domain from a known purl type. - - This method is used to handle the cases where the purl type value is not the git domain but a pre-defined - repo-based type in https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst. - - Note that this method will be updated when there are new pre-defined types as per the PURL specification. - - Parameters - ---------- - purl_type : str - The type field of the PURL. - - Returns - ------- - str | None - The git service domain corresponding to the purl type or None if the purl type is unknown. - """ - known_types = {"github": "github.com", "bitbucket": "bitbucket.org"} - return known_types.get(purl_type, None) - - @staticmethod - def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: - """Return the repository path from the PURL string. - - This method only supports converting a PURL with the following format: - - pkg://[...] - - Where ``type`` could be either: - - The pre-defined repository-based PURL type as defined in - https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst - - - The supprted git service domains (e.g. ``github.com``) defined in ``available_domains``. - - The repository path will be generated with the following format ``https:////``. - - Parameters - ---------- - purl : PackageURL - The parsed PURL to convert to the repository path. - available_domains: list[str] - The list of available domains - - Returns - ------- - str | None - The URL to the repository which the PURL is referring to or None if we cannot convert it. - """ - # TODO: move this method to the repo finder in https://github.com/oracle/macaron/pull/388. - domain = Analyzer.to_domain_from_known_purl_types(purl.type) or ( - purl.type if purl.type in available_domains else None - ) - if not domain: - logger.error("The PURL type of %s is not valid as a repository type.", purl.to_string()) - return None - - if not purl.namespace: - logger.error("Expecting a non-empty namespace from %s.", purl.to_string()) - return None - - # TODO: Handle the version tag and commit digest if they are given in the PURL. - return urlunparse( - ParseResult( - scheme="https", - netloc=domain, - path=os.path.join(purl.namespace, purl.name), - params="", - query="", - fragment="", - ) - ) - class AnalysisTarget(NamedTuple): """Contains the resolved details of a software component to be analyzed. @@ -736,7 +661,7 @@ def to_analysis_target(config: Configuration, available_domains: list[str]) -> A except ValueError as error: raise InvalidPURLError(f"The package url {purl_input_str} is not valid.") from error - converted_repo_path = Analyzer.to_repo_path(parsed_purl, available_domains) + converted_repo_path = repo_finder.to_repo_path(parsed_purl, available_domains) # TODO: If ``converted_repo_path`` is None, this means that the PURL given by the user if not pointing # to a git repository (e.g ``pkg:maven/apache/maven@1.0.0``). Resolving the repository # path from such PURl string will be handled in https://github.com/oracle/macaron/pull/388. @@ -1052,51 +977,34 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: return results - def _resolve_more_dependencies( - self, dependencies: dict[str, DependencyInfo], context: AnalyzeContext, description: str - ) -> None: - """Utilise the Repo Finder to resolve more dependencies to repositories.""" - purl = PackageURL.from_string(context.component.purl) - build_tools = context.dynamic_data["build_spec"]["tools"] - if not purl.type or not build_tools: - logger.debug("No PURL type found for: %s", purl) - else: - # Get actual PURL type based on current build tool - build_tool = build_tools[0] - if len(build_tools) > 1: - logger.debug( - "More than one build tool found for %s, assuming type based on class: %s", - description, - build_tool.__class__, - ) - updated_type = "" - - if isinstance(build_tool, Gradle): - # Currently not a supported PURL type - updated_type = "gradle" - elif isinstance(build_tool, Maven): - updated_type = "maven" - elif isinstance(build_tool, (Pip, Poetry)): - updated_type = "pypi" + def _resolve_more_dependencies(self, dependencies: dict[str, DependencyInfo]) -> None: + """Utilise the Repo Finder to resolve the repositories of more dependencies.""" + for item in dependencies.values(): + if item.get("available") != SCMStatus.MISSING_SCM: + continue + + name = item.get("name") or "" + purl_type = item.get("type_") + if purl_type == "maven": + purl_string = f"pkg:{purl_type}/{item.get('namespace')}/{name}" + elif purl_type == "pypi": + purl_string = f"pkg:{purl_type}/{name.lower().replace('_', '-')}" else: # A type that is not yet supported here - logger.debug("Unsupported PURL type: %s", build_tool.__class__) - - for key, item in dependencies.items(): - if item["available"] != SCMStatus.MISSING_SCM: - continue - dep_namespace, dep_name = key.split(":") - purl_string = f"pkg:{updated_type}/{dep_namespace}/{dep_name}" - if item["version"] != "unspecified": - purl_string = purl_string + "@" + item["version"] - urls = find_repo(purl_string) - item["url"] = DependencyAnalyzer.find_valid_url(urls) - if item["url"] == "": - logger.debug("Failed to find url for purl: %s", purl_string) - else: - # TODO decide how to handle possible duplicates here - item["available"] = SCMStatus.AVAILABLE - item["note"] = "" + logger.debug("Unsupported PURL type: %s", purl_type) + continue + + if item["version"] != "unspecified": + purl_string = purl_string + "@" + item["version"] + + urls = find_repo(PackageURL.from_string(purl_string)) + item["url"] = DependencyAnalyzer.find_valid_url(urls) + if item["url"] == "": + logger.debug("Failed to find url for purl: %s", purl_string) + else: + # TODO decide how to handle possible duplicates here + item["available"] = SCMStatus.AVAILABLE + item["note"] = "" class DuplicateCmpError(DuplicateError): diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index a90f62eef..6e9f66098 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -538,6 +538,7 @@ def get_remote_vcs_url(url: str, clean_up: bool = True) -> str: """ parsed_result = parse_remote_url(url) if not parsed_result: + logger.debug("REJECT: %s", url) return "" url_as_str = urllib.parse.urlunparse(parsed_result) diff --git a/tests/dependency_analyzer/test_dependency_analyzer.py b/tests/dependency_analyzer/test_dependency_analyzer.py index ed9b9c7cb..a3e09f509 100644 --- a/tests/dependency_analyzer/test_dependency_analyzer.py +++ b/tests/dependency_analyzer/test_dependency_analyzer.py @@ -25,21 +25,29 @@ def test_merge_config(self) -> None: auto_deps = { "com.fasterxml.jackson.core:jackson-annotations": DependencyInfo( version="2.14.0-SNAPSHOT", - group="com.fasterxml.jackson.core", + namespace="com.fasterxml.jackson.core", name="jackson-annotations", purl="pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle", url="https://github.com/FasterXML/jackson-annotations", note="", available=SCMStatus.AVAILABLE, + type_="maven", + scheme="pkg", + qualifiers="", + subpath="", ), "com.fasterxml.jackson.core:jackson-core": DependencyInfo( version="2.14.0-SNAPSHOT", - group="com.fasterxml.jackson.core", + namespace="com.fasterxml.jackson.core", name="jackson-core", purl="pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle", url="https://github.com/FasterXML/jackson-core", note="", available=SCMStatus.AVAILABLE, + type_="maven", + scheme="pkg", + qualifiers="", + subpath="", ), } diff --git a/tests/policy_engine/expected_results/policy_report.json b/tests/policy_engine/expected_results/policy_report.json index 59cd0730e..5108356cc 100644 --- a/tests/policy_engine/expected_results/policy_report.json +++ b/tests/policy_engine/expected_results/policy_report.json @@ -8,7 +8,7 @@ "component_violates_policy": [], "component_satisfies_policy": [ [ - "1", + "121", "pkg:github.com/slsa-framework/slsa-verifier@fc50b662fcfeeeb0e97243554b47d9b20b14efac", "auth-provenance" ] diff --git a/tests/slsa_analyzer/test_analyzer.py b/tests/slsa_analyzer/test_analyzer.py index 3b238c211..1ae7d9194 100644 --- a/tests/slsa_analyzer/test_analyzer.py +++ b/tests/slsa_analyzer/test_analyzer.py @@ -42,49 +42,6 @@ def test_resolve_local_path(self) -> None: assert Analyzer._resolve_local_path(self.PARENT_DIR, "././././") == self.PARENT_DIR -@pytest.mark.parametrize( - argnames=("purl", "available_domains", "expect"), - argvalues=[ - # A repo-based PURL cannot have its namespace empty. - ( - PackageURL(type="github", namespace="", name="maven"), - [], - None, - ), - # github and bitbucket are pre-defined types in the PURL specs. - ( - PackageURL(type="github", namespace="apache", name="maven"), - ["github"], - "https://github.com/apache/maven", - ), - ( - PackageURL(type="bitbucket", namespace="snakeyaml", name="snakeyaml"), - ["github"], - "https://bitbucket.org/snakeyaml/snakeyaml", - ), - # Test cases for PURL with git service domain URL as the type. - ( - PackageURL(type="github.com", namespace="apache", name="maven"), - ["github.com", "gitlab.com", "bitbucket.org"], - "https://github.com/apache/maven", - ), - ( - PackageURL(type="bitbucket.org", namespace="snakeyaml", name="snakeyaml"), - ["github.com", "gitlab.com", "bitbucket.org"], - "https://bitbucket.org/snakeyaml/snakeyaml", - ), - ( - PackageURL(type="non-existing", namespace="apache", name="maven"), - ["github.com", "gitlab.com", "bitbucket.org"], - None, - ), - ], -) -def test_to_repo_path(purl: PackageURL, available_domains: list[str], expect: str | None) -> None: - """Test the to repo path method.""" - assert Analyzer.to_repo_path(purl=purl, available_domains=available_domains) == expect - - @pytest.mark.parametrize( ("config", "available_domains", "expect"), [ From e56ed6cfba0854620c2a29ee4915266ad9ebac50 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 30 Aug 2023 09:33:13 +1000 Subject: [PATCH 08/31] chore: updated parser name Signed-off-by: Ben Selwyn-Smith --- src/macaron/__main__.py | 10 +++++----- src/macaron/repo_finder/repo_finder_java.py | 2 +- src/macaron/slsa_analyzer/git_url.py | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index 0a813d2a8..84c542cca 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -32,9 +32,9 @@ def analyze_slsa_levels_single(analyzer_single_args: argparse.Namespace) -> None # We don't mention --config-path as a possible option in this log message as it going to be move soon. # See: https://github.com/oracle/macaron/issues/417 logger.error( - "Analysis target missing. Please provide a package url (PURL) and/or repo path. " - + "Examples of a PURL can be seen at https://github.com/package-url/purl-spec: " - + "pkg:github/micronaut-projects/micronaut-core." + """Analysis target missing. Please provide a package url (PURL) and/or repo path. + Examples of a PURL can be seen at https://github.com/package-url/purl-spec: + pkg:github/micronaut-projects/micronaut-core.""" ) sys.exit(os.EX_USAGE) @@ -376,8 +376,8 @@ def main(argv: list[str] | None = None) -> None: vp_parser = sub_parser.add_parser(name="verify-policy") vp_group = vp_parser.add_mutually_exclusive_group(required=True) - # Verify the Repo Finder - sub_parser.add_parser(name="verify-repo-finder") + # Test the Repo Finder + sub_parser.add_parser(name="test-repo-finder") vp_parser.add_argument("-d", "--database", required=True, type=str, help="Path to the database.") vp_group.add_argument("-f", "--file", type=str, help="Path to the Datalog policy.") diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 225729efb..e7e0d802d 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -75,7 +75,7 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: urls = self.read_metadata(pom) if urls: - logger.debug("Found %s urls.", len(urls)) + logger.debug("Found %s urls: %s", len(urls), urls) yield from iter(urls) if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index 6e9f66098..058030775 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -536,10 +536,14 @@ def get_remote_vcs_url(url: str, clean_up: bool = True) -> str: str The remote url to the repo or empty if the url is invalid. """ + if len(url) == 1: + return "" + parsed_result = parse_remote_url(url) if not parsed_result: logger.debug("REJECT: %s", url) return "" + logger.debug("ACCEPT: %s", url) url_as_str = urllib.parse.urlunparse(parsed_result) From d274dd0822602113e318adfc41da7c99d422903d Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 31 Aug 2023 08:58:22 +1000 Subject: [PATCH 09/31] chore: minor fix Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 07eda46fe..084cdfca6 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -997,8 +997,7 @@ def _resolve_more_dependencies(self, dependencies: dict[str, DependencyInfo]) -> if item["version"] != "unspecified": purl_string = purl_string + "@" + item["version"] - urls = find_repo(PackageURL.from_string(purl_string)) - item["url"] = DependencyAnalyzer.find_valid_url(urls) + item["url"] = find_repo(PackageURL.from_string(purl_string)) if item["url"] == "": logger.debug("Failed to find url for purl: %s", purl_string) else: From 0a7096fcb69b011e2cd6e4b2f5e9db72b6875c30 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 31 Aug 2023 09:18:27 +1000 Subject: [PATCH 10/31] chore: extended docstring of repo finder Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 3904538c1..411ee3e7e 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -1,7 +1,36 @@ # Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This module contains the logic for using/calling the different repo finders.""" +""" +This module contains the logic for using/calling the different repo finders. + +Input +----- +The entry point of the repo finder depends on the type of PURL being analyzed. +- If passing a PURL representing an artifact, the ``find_repo`` function in this file should be called. +- If passing a PURL representing a repository, the ``to_repo_path`` function in this file should be called. + +Artifact PURLs +-------------- +For artifact PURLs, the PURL type determines how the repositories are searched for. +Currently, for Maven PURLs, SCM meta data is retrieved from the matching POM retrieved from Maven Central (or +other configured location). + +For Python, .NET, Rust, and NodeJS type PURLs, Google's Open Source Insights API is used to find the meta data. + +In either case, any repository links are extracted from the meta data, then checked for validity via +``DependencyAnalyzer::find_valid_url`` which accepts URLs that point to a Github repository or similar. + +Repository PURLs +---------------- +For repository PURLs, the type is checked against the configured valid domains, and accepted or rejected based +on that data. + +Result +------ +If all goes well, a repository URL that matches the initial artifact or repository PURL will be returned for +analysis. +""" import logging import os From 0a8b665a72a090b22674f73ea6cdb321f84dadd7 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 12:35:49 +1000 Subject: [PATCH 11/31] chore: Addressed PR feedback. - Moved dependency resolution functions to dependency_resolver.py - Moved find_valid_url to repo_finder to avoid circular dependencies - Added URL for deps.dev API documentation as comment Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 5 +- .../dependency_resolver.py | 166 +++++++++++++++--- src/macaron/repo_finder/repo_finder.py | 32 +++- .../repo_finder/repo_finder_deps_dev.py | 3 +- src/macaron/slsa_analyzer/analyzer.py | 137 +-------------- 5 files changed, 175 insertions(+), 168 deletions(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 6029b058e..b52aad30d 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -14,6 +14,7 @@ from macaron.dependency_analyzer.dependency_resolver import DependencyAnalyzer, DependencyInfo from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus +from macaron.repo_finder.repo_finder import find_valid_url logger: logging.Logger = logging.getLogger(__name__) @@ -160,7 +161,7 @@ def convert_components_to_artifacts( Returns ------- dict - A dictionary where dependency artifacts are grouped based on "artifactId:groupId". + A dictionary where dependency artifacts are grouped based on "groupId:artifactId". """ all_versions: dict[str, list[DependencyInfo]] = {} # Stores all the versions of dependencies for debugging. latest_deps: dict[str, DependencyInfo] = {} # Stores the latest version of dependencies. @@ -199,7 +200,7 @@ def convert_components_to_artifacts( ) else: # Find a valid URL. - item["url"] = DependencyAnalyzer.find_valid_url( + item["url"] = find_valid_url( link.get("url") for link in component.get("externalReferences") # type: ignore ) diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index c76b64966..fc8e34d23 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -4,17 +4,24 @@ """This module processes and collects the dependencies to be processed by Macaron.""" import logging +import os +import subprocess # nosec B404 from abc import ABC, abstractmethod from collections.abc import Iterable from enum import Enum -from typing import TypedDict +from pathlib import Path +from typing import Any, TypedDict +from packageurl import PackageURL from packaging import version +from macaron.config.defaults import defaults +from macaron.config.global_config import global_config from macaron.config.target_config import Configuration from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus -from macaron.slsa_analyzer.git_url import get_remote_vcs_url, get_repo_full_name_from_url +from macaron.repo_finder.repo_finder import find_repo, find_valid_url +from macaron.slsa_analyzer.git_url import get_repo_full_name_from_url logger: logging.Logger = logging.getLogger(__name__) @@ -168,31 +175,6 @@ def add_latest_version( except ValueError as error: logger.error("Could not parse dependency version number: %s", error) - @staticmethod - def find_valid_url(urls: Iterable[str]) -> str: - """Find a valid URL from the provided URLs. - - Parameters - ---------- - urls : Iterable[str] - An Iterable object containing urls. - - Returns - ------- - str - A valid URL or empty if it can't find any valid URL. - """ - vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} - - # To avoid non-deterministic results we sort the URLs. - vcs_list = sorted(vcs_set) - - if len(vcs_list) < 1: - return "" - - # Report the first valid URL. - return vcs_list.pop() - @staticmethod def merge_configs( config_deps: list[Configuration], resolved_deps: dict[str, DependencyInfo] @@ -283,6 +265,136 @@ def tool_valid(tool: str) -> bool: return False return True + @staticmethod + def resolve_dependencies(main_ctx: Any, sbom_path: str) -> dict[str, DependencyInfo]: + """Resolve the dependencies of the main target repo. + + Parameters + ---------- + main_ctx : Any (AnalyzeContext) + The context of object of the target repository. + sbom_path: str + The path to the SBOM. + + Returns + ------- + dict[str, DependencyInfo] + A dictionary where artifacts are grouped based on ``artifactId:groupId``. + """ + if sbom_path: + logger.info("Getting the dependencies from the SBOM defined at %s.", sbom_path) + # Import here to avoid circular dependency + # pylint: disable=import-outside-toplevel, cyclic-import + from macaron.dependency_analyzer.cyclonedx import get_deps_from_sbom + + return get_deps_from_sbom(sbom_path) + + deps_resolved: dict[str, DependencyInfo] = {} + + build_tools = main_ctx.dynamic_data["build_spec"]["tools"] + if not build_tools: + logger.info("Unable to find any valid build tools.") + return {} + + # Grab dependencies for each build tool, collate all into the deps_resolved + for build_tool in build_tools: + try: + dep_analyzer = build_tool.get_dep_analyzer(main_ctx.component.repository.fs_path) + except DependencyAnalyzerError as error: + logger.error("Unable to find a dependency analyzer for %s: %s", build_tool.name, error) + return {} + + if isinstance(dep_analyzer, NoneDependencyAnalyzer): + logger.info( + "Dependency analyzer is not available for %s", + build_tool.name, + ) + return {} + + # Start resolving dependencies. + logger.info( + "Running %s version %s dependency analyzer on %s", + dep_analyzer.tool_name, + dep_analyzer.tool_version, + main_ctx.component.repository.fs_path, + ) + + log_path = os.path.join( + global_config.build_log_path, + f"{main_ctx.component.report_file_name}.{dep_analyzer.tool_name}.log", + ) + + # Clean up existing SBOM files. + dep_analyzer.remove_sboms(main_ctx.component.repository.fs_path) + + commands = dep_analyzer.get_cmd() + working_dirs: Iterable[Path] = build_tool.get_build_dirs(main_ctx.component.repository.fs_path) + for working_dir in working_dirs: + # Get the absolute path to use as the working dir in the subprocess. + working_dir = Path(main_ctx.component.repository.fs_path).joinpath(working_dir) + + try: + # Suppressing Bandit's B603 report because the repo paths are validated. + analyzer_output = subprocess.run( # nosec B603 + commands, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True, + cwd=str(working_dir), + timeout=defaults.getint("dependency.resolver", "timeout", fallback=1200), + ) + with open(log_path, mode="a", encoding="utf-8") as log_file: + log_file.write(analyzer_output.stdout.decode("utf-8")) + + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + logger.error(error) + with open(log_path, mode="a", encoding="utf-8") as log_file: + log_file.write(error.output.decode("utf-8")) + except FileNotFoundError as error: + logger.error(error) + + # We collect the generated SBOM as a best effort, even if the build exits with errors. + # TODO: add improvements to help the SBOM build succeed as much as possible. + deps_resolved |= dep_analyzer.collect_dependencies(str(working_dir)) + + logger.info("Stored dependency resolver log for %s to %s.", dep_analyzer.tool_name, log_path) + + # Use repo finder to find more repositories to analyze. + if defaults.getboolean("repofinder", "find_repos"): + DependencyAnalyzer._resolve_more_dependencies(deps_resolved) + + return deps_resolved + + @staticmethod + def _resolve_more_dependencies(dependencies: dict[str, DependencyInfo]) -> None: + """Utilise the Repo Finder to resolve the repositories of more dependencies.""" + for item in dependencies.values(): + if item.get("available") != SCMStatus.MISSING_SCM: + continue + + name = item.get("name") or "" + purl_type = item.get("type_") + if purl_type == "maven": + purl_string = f"pkg:{purl_type}/{item.get('namespace')}/{name}" + elif purl_type == "pypi": + purl_string = f"pkg:{purl_type}/{name.lower().replace('_', '-')}" + else: + # A type that is not yet supported here + logger.debug("Unsupported PURL type: %s", purl_type) + continue + + if item["version"] != "unspecified": + purl_string = purl_string + "@" + item["version"] + + urls = find_repo(PackageURL.from_string(purl_string)) + item["url"] = find_valid_url(urls) + if item["url"] == "": + logger.debug("Failed to find url for purl: %s", purl_string) + else: + # TODO decide how to handle possible duplicates here + item["available"] = SCMStatus.AVAILABLE + item["note"] = "" + class NoneDependencyAnalyzer(DependencyAnalyzer): """This class is used to implement an empty dependency analyzers.""" diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 411ee3e7e..c7c4c56f1 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -34,15 +34,16 @@ import logging import os +from collections.abc import Iterable from urllib.parse import ParseResult, urlunparse from packageurl import PackageURL from macaron.config.defaults import defaults -from macaron.dependency_analyzer import DependencyAnalyzer from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev from macaron.repo_finder.repo_finder_java import JavaRepoFinder +from macaron.slsa_analyzer.git_url import get_remote_vcs_url logger: logging.Logger = logging.getLogger(__name__) @@ -76,8 +77,8 @@ def find_repo(purl: PackageURL) -> str: # Call Repo Finder and return first valid URL logger.debug("Analyzing %s with Repo Finder: %s", purl.to_string(), repo_finder.__class__) - found_urls = repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") - return DependencyAnalyzer.find_valid_url(found_urls) + urls = repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") + return find_valid_url(urls) def to_domain_from_known_purl_types(purl_type: str) -> str | None: @@ -150,3 +151,28 @@ def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: fragment="", ) ) + + +def find_valid_url(urls: Iterable[str]) -> str: + """Find a valid URL from the provided URLs. + + Parameters + ---------- + urls : Iterable[str] + An Iterable object containing urls. + + Returns + ------- + str + A valid URL or empty if it can't find any valid URL. + """ + vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} + + # To avoid non-deterministic results we sort the URLs. + vcs_list = sorted(vcs_set) + + if len(vcs_list) < 1: + return "" + + # Report the first valid URL. + return vcs_list.pop() diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 4385f9af5..94e84a0a1 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -159,7 +159,7 @@ def read_metadata(self, metadata: str) -> list[str]: return list(parsed["links"]) def create_type_specific_url(self, namespace: str, name: str) -> str: - """Create a url for the deps.dev API based on the package type. + """Create a URL for the deps.dev API based on the package type. Parameters ---------- @@ -176,6 +176,7 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: namespace = encode(namespace) name = encode(name) + # See https://docs.deps.dev/api/v3alpha/ match self.type: case "pypi": package_name = name.lower().replace("_", "-") diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 084cdfca6..f02c12216 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -5,9 +5,7 @@ import logging import os -import subprocess # nosec B404 import sys -from collections.abc import Iterable from datetime import datetime, timezone from pathlib import Path from typing import NamedTuple @@ -19,23 +17,15 @@ from sqlalchemy.orm import Session from macaron import __version__ -from macaron.config.defaults import defaults from macaron.config.global_config import global_config from macaron.config.target_config import Configuration from macaron.database.database_manager import DatabaseManager, get_db_manager, get_db_session from macaron.database.table_definitions import Analysis, Component, Repository -from macaron.dependency_analyzer import ( - DependencyAnalyzer, - DependencyAnalyzerError, - DependencyInfo, - NoneDependencyAnalyzer, -) -from macaron.dependency_analyzer.cyclonedx import get_deps_from_sbom +from macaron.dependency_analyzer import DependencyAnalyzer, DependencyInfo from macaron.errors import CloneError, DuplicateError, InvalidPURLError, PURLNotFoundError, RepoCheckOutError from macaron.output_reporter.reporter import FileReporter from macaron.output_reporter.results import Record, Report, SCMStatus from macaron.repo_finder import repo_finder -from macaron.repo_finder.repo_finder import find_repo from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.build_tool import BUILD_TOOLS @@ -166,7 +156,7 @@ def run(self, user_config: dict, sbom_path: str = "", skip_deps: bool = False) - if skip_deps: logger.info("Skipping automatic dependency analysis...") else: - deps_resolved = self.resolve_dependencies(main_record.context, sbom_path) + deps_resolved = DependencyAnalyzer.resolve_dependencies(main_record.context, sbom_path) # Merge the automatically resolved dependencies with the manual configuration. deps_config = DependencyAnalyzer.merge_configs(deps_config, deps_resolved) @@ -254,101 +244,6 @@ def generate_reports(self, report: Report) -> None: for reporter in self.reporters: reporter.generate(output_target_path, report) - def resolve_dependencies(self, main_ctx: AnalyzeContext, sbom_path: str) -> dict[str, DependencyInfo]: - """Resolve the dependencies of the main target repo. - - Parameters - ---------- - main_ctx : AnalyzeContext - The context of object of the target repository. - sbom_path: str - The path to the SBOM. - - Returns - ------- - dict[str, DependencyInfo] - A dictionary where artifacts are grouped based on ``artifactId:groupId``. - """ - if sbom_path: - logger.info("Getting the dependencies from the SBOM defined at %s.", sbom_path) - return get_deps_from_sbom(sbom_path) - - deps_resolved: dict[str, DependencyInfo] = {} - - build_tools = main_ctx.dynamic_data["build_spec"]["tools"] - if not build_tools: - logger.info("Unable to find any valid build tools.") - return {} - - # Grab dependencies for each build tool, collate all into the deps_resolved - for build_tool in build_tools: - try: - dep_analyzer = build_tool.get_dep_analyzer(main_ctx.component.repository.fs_path) - except DependencyAnalyzerError as error: - logger.error("Unable to find a dependency analyzer for %s: %s", build_tool.name, error) - return {} - - if isinstance(dep_analyzer, NoneDependencyAnalyzer): - logger.info( - "Dependency analyzer is not available for %s", - build_tool.name, - ) - return {} - - # Start resolving dependencies. - logger.info( - "Running %s version %s dependency analyzer on %s", - dep_analyzer.tool_name, - dep_analyzer.tool_version, - main_ctx.component.repository.fs_path, - ) - - log_path = os.path.join( - global_config.build_log_path, - f"{main_ctx.component.report_file_name}.{dep_analyzer.tool_name}.log", - ) - - # Clean up existing SBOM files. - dep_analyzer.remove_sboms(main_ctx.component.repository.fs_path) - - commands = dep_analyzer.get_cmd() - working_dirs: Iterable[Path] = build_tool.get_build_dirs(main_ctx.component.repository.fs_path) - for working_dir in working_dirs: - # Get the absolute path to use as the working dir in the subprocess. - working_dir = Path(main_ctx.component.repository.fs_path).joinpath(working_dir) - - try: - # Suppressing Bandit's B603 report because the repo paths are validated. - analyzer_output = subprocess.run( # nosec B603 - commands, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=True, - cwd=str(working_dir), - timeout=defaults.getint("dependency.resolver", "timeout", fallback=1200), - ) - with open(log_path, mode="a", encoding="utf-8") as log_file: - log_file.write(analyzer_output.stdout.decode("utf-8")) - - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: - logger.error(error) - with open(log_path, mode="a", encoding="utf-8") as log_file: - log_file.write(error.output.decode("utf-8")) - except FileNotFoundError as error: - logger.error(error) - - # We collect the generated SBOM as a best effort, even if the build exits with errors. - # TODO: add improvements to help the SBOM build succeed as much as possible. - deps_resolved |= dep_analyzer.collect_dependencies(str(working_dir)) - - logger.info("Stored dependency resolver log for %s to %s.", dep_analyzer.tool_name, log_path) - - # Use repo finder to find more repositories to analyze. - if defaults.getboolean("repofinder", "find_repos"): - self._resolve_more_dependencies(deps_resolved) - - return deps_resolved - def run_single( self, config: Configuration, @@ -977,34 +872,6 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: return results - def _resolve_more_dependencies(self, dependencies: dict[str, DependencyInfo]) -> None: - """Utilise the Repo Finder to resolve the repositories of more dependencies.""" - for item in dependencies.values(): - if item.get("available") != SCMStatus.MISSING_SCM: - continue - - name = item.get("name") or "" - purl_type = item.get("type_") - if purl_type == "maven": - purl_string = f"pkg:{purl_type}/{item.get('namespace')}/{name}" - elif purl_type == "pypi": - purl_string = f"pkg:{purl_type}/{name.lower().replace('_', '-')}" - else: - # A type that is not yet supported here - logger.debug("Unsupported PURL type: %s", purl_type) - continue - - if item["version"] != "unspecified": - purl_string = purl_string + "@" + item["version"] - - item["url"] = find_repo(PackageURL.from_string(purl_string)) - if item["url"] == "": - logger.debug("Failed to find url for purl: %s", purl_string) - else: - # TODO decide how to handle possible duplicates here - item["available"] = SCMStatus.AVAILABLE - item["note"] = "" - class DuplicateCmpError(DuplicateError): """This class is used for duplicated software component errors.""" From 7745019361c01e8ddaf98c500da7d7e565b3fcff Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 12:52:02 +1000 Subject: [PATCH 12/31] chore: moved repo finder integration test to new file Signed-off-by: Ben Selwyn-Smith --- scripts/dev_scripts/integration_tests.sh | 3 +- .../dev_scripts/integration_tests_docker.sh | 2 +- src/macaron/__main__.py | 32 ------------- tests/e2e/repo_finder/__init__.py | 2 + tests/e2e/repo_finder/repo_finder.py | 45 +++++++++++++++++++ 5 files changed, 50 insertions(+), 34 deletions(-) create mode 100644 tests/e2e/repo_finder/__init__.py create mode 100644 tests/e2e/repo_finder/repo_finder.py diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index d43d45c1e..783c185de 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -9,6 +9,7 @@ HOMEDIR=$2 RESOURCES=$WORKSPACE/src/macaron/resources COMPARE_DEPS=$WORKSPACE/tests/dependency_analyzer/compare_dependencies.py COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py +TEST_REPO_FINDER=$WORKSPACE/tests/e2e/repo_finder/repo_finder.py RUN_MACARON="python -m macaron -o $WORKSPACE/output" RESULT_CODE=0 @@ -537,7 +538,7 @@ fi echo -e "\n----------------------------------------------------------------------------------" echo "Testing Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -macaron -v test-repo-finder +python $COMPARE_JSON_OUT || log_fail if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index bb78d1431..27d18896f 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -149,7 +149,7 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail echo -e "\n----------------------------------------------------------------------------------" echo "Test Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -$RUN_MACARON_SCRIPT -v test-repo-finder +python $COMPARE_JSON_OUT || log_fail if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index 84c542cca..001241882 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -18,7 +18,6 @@ from macaron.output_reporter.reporter import HTMLReporter, JSONReporter, PolicyReporter from macaron.parsers.yaml.loader import YamlLoader from macaron.policy_engine.policy_engine import run_policy_engine, show_prelude -from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev from macaron.slsa_analyzer.analyzer import Analyzer from macaron.slsa_analyzer.git_service import GIT_SERVICES from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES @@ -159,31 +158,6 @@ def verify_policy(verify_policy_args: argparse.Namespace) -> int: return os.EX_USAGE -def test_repo_finder() -> int: - """Test the functionality of the remote API calls used by the repo finder. - - Functionality relating to Java artifacts is not verified for two reasons: - - It is extremely unlikely that Maven central will change its API or cease operation in the near future. - - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. - """ - # Test deps.dev API for a Python package - repo_finder = RepoFinderDepsDev("pypi") - urls = [] - # Without version - urls.append(repo_finder.create_urls("", "packageurl-python", "")) - # With version - urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) - for url in urls: - logger.debug("Testing: %s", url[0]) - metadata = repo_finder.retrieve_metadata(url[0]) - if not metadata: - return os.EX_UNAVAILABLE - links = repo_finder.read_metadata(metadata) - if not links: - return os.EX_UNAVAILABLE - return os.EX_OK - - def perform_action(action_args: argparse.Namespace) -> None: """Perform the indicated action of Macaron.""" match action_args.action: @@ -195,9 +169,6 @@ def perform_action(action_args: argparse.Namespace) -> None: case "verify-policy": sys.exit(verify_policy(action_args)) - case "test-repo-finder": - sys.exit(test_repo_finder()) - case "analyze": # Check that the GitHub token is enabled. gh_token = os.environ.get("GITHUB_TOKEN") @@ -376,9 +347,6 @@ def main(argv: list[str] | None = None) -> None: vp_parser = sub_parser.add_parser(name="verify-policy") vp_group = vp_parser.add_mutually_exclusive_group(required=True) - # Test the Repo Finder - sub_parser.add_parser(name="test-repo-finder") - vp_parser.add_argument("-d", "--database", required=True, type=str, help="Path to the database.") vp_group.add_argument("-f", "--file", type=str, help="Path to the Datalog policy.") vp_group.add_argument("-s", "--show-prelude", action="store_true", help="Show policy prelude.") diff --git a/tests/e2e/repo_finder/__init__.py b/tests/e2e/repo_finder/__init__.py new file mode 100644 index 000000000..19aeac023 --- /dev/null +++ b/tests/e2e/repo_finder/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/e2e/repo_finder/repo_finder.py b/tests/e2e/repo_finder/repo_finder.py new file mode 100644 index 000000000..015940025 --- /dev/null +++ b/tests/e2e/repo_finder/repo_finder.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the functionality of the repo finder's remote API calls.""" + +import logging +import os + +from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev + +logger: logging.Logger = logging.getLogger(__name__) + +# Set logging debug level. +logger.setLevel(logging.DEBUG) + + +def test_repo_finder() -> int: + """Test the functionality of the remote API calls used by the repo finder. + + Functionality relating to Java artifacts is not verified for two reasons: + - It is extremely unlikely that Maven central will change its API or cease operation in the near future. + - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. + """ + # Test deps.dev API for a Python package + repo_finder = RepoFinderDepsDev("pypi") + urls = [] + # Without version + urls.append(repo_finder.create_urls("", "packageurl-python", "")) + # With version + urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) + for url in urls: + logger.debug("Testing: %s", url[0]) + metadata = repo_finder.retrieve_metadata(url[0]) + if not metadata: + return os.EX_UNAVAILABLE + links = repo_finder.read_metadata(metadata) + if not links: + return os.EX_UNAVAILABLE + return os.EX_OK + + +if __name__ == "__main__": + test_repo_finder() From 60a5cc7058be8cb54e05809c86d7d7874940dda0 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 13:26:04 +1000 Subject: [PATCH 13/31] chore: try to derive the SBOM component type Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index b52aad30d..135eaece8 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -9,6 +9,8 @@ from collections.abc import Iterable from pathlib import Path +from packageurl import PackageURL + from macaron.config.defaults import defaults from macaron.config.global_config import global_config from macaron.dependency_analyzer.dependency_resolver import DependencyAnalyzer, DependencyInfo @@ -168,7 +170,13 @@ def convert_components_to_artifacts( url_to_artifact: dict[str, set] = {} # Used to detect artifacts that have similar repos. for component in components: try: + # TODO make this function language agnostic when CycloneDX SBOM processing also is. + # See https://github.com/oracle/macaron/issues/464 key = f"{component.get('group')}:{component.get('name')}" + # The component's purl is optional, but we must make an attempt to retrieve the purl type. + component_type = "" + if component.get("purl"): + component_type = PackageURL.from_string(str(component.get("purl"))).type # According to PEP-0589 all keys must be present in a TypedDict. # See https://peps.python.org/pep-0589/#totality item = DependencyInfo( @@ -229,7 +237,7 @@ def get_deps_from_sbom(sbom_path: str | Path) -> dict[str, DependencyInfo]: Returns ------- - A dictionary where dependency artifacts are grouped based on "artifactId:groupId". + A dictionary where dependency artifacts are grouped based on "groupId:artifactId". """ return convert_components_to_artifacts( get_dep_components( From 64a7472bd28da73dfffc01e9cef97b5799d09dd4 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 14:17:37 +1000 Subject: [PATCH 14/31] chore: Add PURL to DependencyInfo; Try to retrieve PURL from SBOM for use with Repo Finder. Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 23 +++++++------- .../dependency_resolver.py | 30 ++++--------------- .../test_dependency_analyzer.py | 22 ++++---------- 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 135eaece8..f44d588cf 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -173,17 +173,20 @@ def convert_components_to_artifacts( # TODO make this function language agnostic when CycloneDX SBOM processing also is. # See https://github.com/oracle/macaron/issues/464 key = f"{component.get('group')}:{component.get('name')}" - # The component's purl is optional, but we must make an attempt to retrieve the purl type. - component_type = "" if component.get("purl"): - component_type = PackageURL.from_string(str(component.get("purl"))).type + purl = PackageURL.from_string(str(component.get("purl"))) + else: + # TODO remove maven assumption when existence of the component's purl is handled + # See https://github.com/oracle/macaron/issues/464 + purl_string = f"pkg:maven/{component.get('group')}/{component.get('name')}" + if component.get("version"): + purl_string = f"{purl_string}@{component.get('version')}" + purl = PackageURL.from_string(purl_string) + # According to PEP-0589 all keys must be present in a TypedDict. # See https://peps.python.org/pep-0589/#totality item = DependencyInfo( - version=component.get("version") or "", - namespace=component.get("group") or "", - name=component.get("name") or "", - purl=component.get("purl") or "", + purl=purl, url="", note="", available=SCMStatus.AVAILABLE, @@ -196,10 +199,10 @@ def convert_components_to_artifacts( # IN case of a build error, we use this as a heuristic to avoid analyzing # submodules that produce development artifacts in the same repo. if ( - "snapshot" - in (item.get("version") or "").lower() # or "" is not necessary but mypy produces a FP otherwise. + "snapshot" in (purl.version or "").lower() + # or "" is not necessary but mypy produces a FP otherwise. and root_component - and item.get("namespace") == root_component.get("group") + and purl.namespace == root_component.get("group") ): continue logger.debug( diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index fc8e34d23..3330cb5cd 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -20,7 +20,7 @@ from macaron.config.target_config import Configuration from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus -from macaron.repo_finder.repo_finder import find_repo, find_valid_url +from macaron.repo_finder.repo_finder import find_repo from macaron.slsa_analyzer.git_url import get_repo_full_name_from_url logger: logging.Logger = logging.getLogger(__name__) @@ -36,10 +36,7 @@ class DependencyTools(str, Enum): class DependencyInfo(TypedDict): """The information of a resolved dependency.""" - version: str - namespace: str - name: str - purl: str + purl: PackageURL url: str note: str available: SCMStatus @@ -167,8 +164,8 @@ def add_latest_version( else: try: if ( - (latest_version := latest_value.get("version")) - and (item_version := item.get("version")) + (latest_version := latest_value.get("purl").version) # type: ignore + and (item_version := item.get("purl").version) # type: ignore and version.Version(latest_version) < version.Version(item_version) ): latest_deps[key] = item @@ -372,24 +369,9 @@ def _resolve_more_dependencies(dependencies: dict[str, DependencyInfo]) -> None: if item.get("available") != SCMStatus.MISSING_SCM: continue - name = item.get("name") or "" - purl_type = item.get("type_") - if purl_type == "maven": - purl_string = f"pkg:{purl_type}/{item.get('namespace')}/{name}" - elif purl_type == "pypi": - purl_string = f"pkg:{purl_type}/{name.lower().replace('_', '-')}" - else: - # A type that is not yet supported here - logger.debug("Unsupported PURL type: %s", purl_type) - continue - - if item["version"] != "unspecified": - purl_string = purl_string + "@" + item["version"] - - urls = find_repo(PackageURL.from_string(purl_string)) - item["url"] = find_valid_url(urls) + item["url"] = find_repo(item.get("purl")) # type: ignore if item["url"] == "": - logger.debug("Failed to find url for purl: %s", purl_string) + logger.debug("Failed to find url for purl: %s", item.get("purl")) else: # TODO decide how to handle possible duplicates here item["available"] = SCMStatus.AVAILABLE diff --git a/tests/dependency_analyzer/test_dependency_analyzer.py b/tests/dependency_analyzer/test_dependency_analyzer.py index a3e09f509..873abdf17 100644 --- a/tests/dependency_analyzer/test_dependency_analyzer.py +++ b/tests/dependency_analyzer/test_dependency_analyzer.py @@ -7,6 +7,8 @@ from pathlib import Path +from packageurl import PackageURL + from macaron.config.target_config import TARGET_CONFIG_SCHEMA, Configuration from macaron.dependency_analyzer import DependencyAnalyzer, DependencyInfo from macaron.output_reporter.scm import SCMStatus @@ -22,32 +24,20 @@ class TestDependencyAnalyzer(MacaronTestCase): def test_merge_config(self) -> None: """Test merging the manual and automatically resolved configurations.""" # Mock automatically resolved dependencies. + purl_string_1 = "pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle" + purl_string_2 = "pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle" auto_deps = { "com.fasterxml.jackson.core:jackson-annotations": DependencyInfo( - version="2.14.0-SNAPSHOT", - namespace="com.fasterxml.jackson.core", - name="jackson-annotations", - purl="pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle", + purl=PackageURL.from_string(purl_string_1), url="https://github.com/FasterXML/jackson-annotations", note="", available=SCMStatus.AVAILABLE, - type_="maven", - scheme="pkg", - qualifiers="", - subpath="", ), "com.fasterxml.jackson.core:jackson-core": DependencyInfo( - version="2.14.0-SNAPSHOT", - namespace="com.fasterxml.jackson.core", - name="jackson-core", - purl="pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle", + purl=PackageURL.from_string(purl_string_2), url="https://github.com/FasterXML/jackson-core", note="", available=SCMStatus.AVAILABLE, - type_="maven", - scheme="pkg", - qualifiers="", - subpath="", ), } From 556514e6a41646a116e5e0bd8b35024efca7c63c Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 14:27:04 +1000 Subject: [PATCH 15/31] chore: renaming of deps.dev files Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 4 ++-- src/macaron/repo_finder/repo_finder_deps_dev.py | 4 ++-- tests/e2e/repo_finder/repo_finder.py | 4 ++-- .../{repo_finder_dd => repo_finder_deps_dev}/__init__.py | 0 .../test_repo_finder_dd.py | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) rename tests/repo_finder/{repo_finder_dd => repo_finder_deps_dev}/__init__.py (100%) rename tests/repo_finder/{repo_finder_dd => repo_finder_deps_dev}/test_repo_finder_dd.py (74%) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index c7c4c56f1..426a4738e 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -41,7 +41,7 @@ from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_finder_java import JavaRepoFinder from macaron.slsa_analyzer.git_url import get_remote_vcs_url @@ -70,7 +70,7 @@ def find_repo(purl: PackageURL) -> str: "cargo", "npm", ]: - repo_finder = RepoFinderDepsDev(purl.type) + repo_finder = DepsDevRepoFinder(purl.type) else: logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl.to_string()) return "" diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 94e84a0a1..a30f56312 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -15,8 +15,8 @@ logger: logging.Logger = logging.getLogger(__name__) -class RepoFinderDepsDev(BaseRepoFinder): - """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev (DD).""" +class DepsDevRepoFinder(BaseRepoFinder): + """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev.""" def __init__(self, purl_type: str) -> None: """Initialise the deps.dev repository finder instance. diff --git a/tests/e2e/repo_finder/repo_finder.py b/tests/e2e/repo_finder/repo_finder.py index 015940025..90d71acb4 100644 --- a/tests/e2e/repo_finder/repo_finder.py +++ b/tests/e2e/repo_finder/repo_finder.py @@ -8,7 +8,7 @@ import logging import os -from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder logger: logging.Logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def test_repo_finder() -> int: - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. """ # Test deps.dev API for a Python package - repo_finder = RepoFinderDepsDev("pypi") + repo_finder = DepsDevRepoFinder("pypi") urls = [] # Without version urls.append(repo_finder.create_urls("", "packageurl-python", "")) diff --git a/tests/repo_finder/repo_finder_dd/__init__.py b/tests/repo_finder/repo_finder_deps_dev/__init__.py similarity index 100% rename from tests/repo_finder/repo_finder_dd/__init__.py rename to tests/repo_finder/repo_finder_deps_dev/__init__.py diff --git a/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py b/tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py similarity index 74% rename from tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py rename to tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py index 0c945d5c8..26157e8fa 100644 --- a/tests/repo_finder/repo_finder_dd/test_repo_finder_dd.py +++ b/tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py @@ -4,12 +4,12 @@ """ This module tests the python dd repo finder. """ -from macaron.repo_finder.repo_finder_deps_dev import RepoFinderDepsDev +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder -def test_repo_finder_dd() -> None: +def test_repo_finder_deps_dev() -> None: """Test the functions of the repo finder.""" - repo_finder = RepoFinderDepsDev("pypi") + repo_finder = DepsDevRepoFinder("pypi") assert ( repo_finder.create_type_specific_url("", "packageurl-python") == "https://api.deps.dev/v3alpha/systems/pypi/packages/packageurl-python" From 3ad9c4af40b1d03f9fb184bce1ea652e40b073e5 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 15:07:25 +1000 Subject: [PATCH 16/31] chore: added integration tests for more languages Signed-off-by: Ben Selwyn-Smith --- .../repo_finder/repo_finder_deps_dev.py | 8 +++- src/macaron/slsa_analyzer/git_url.py | 5 --- tests/e2e/repo_finder/repo_finder.py | 45 ++++++++++++------- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index a30f56312..eb447d5d8 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -156,7 +156,11 @@ def read_metadata(self, metadata: str) -> list[str]: logger.debug("Metadata had no URLs: %s", parsed["versionKey"]) return [] - return list(parsed["links"]) + result = [] + for item in parsed["links"]: + result.append(item.get("url")) + + return result def create_type_specific_url(self, namespace: str, name: str) -> str: """Create a URL for the deps.dev API based on the package type. @@ -182,7 +186,7 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: package_name = name.lower().replace("_", "-") case "npm": if namespace: - package_name = f"%40{namespace}%2F{name}" + package_name = f"{namespace}%2F{name}" else: package_name = name case "nuget" | "cargo": diff --git a/src/macaron/slsa_analyzer/git_url.py b/src/macaron/slsa_analyzer/git_url.py index 058030775..a90f62eef 100644 --- a/src/macaron/slsa_analyzer/git_url.py +++ b/src/macaron/slsa_analyzer/git_url.py @@ -536,14 +536,9 @@ def get_remote_vcs_url(url: str, clean_up: bool = True) -> str: str The remote url to the repo or empty if the url is invalid. """ - if len(url) == 1: - return "" - parsed_result = parse_remote_url(url) if not parsed_result: - logger.debug("REJECT: %s", url) return "" - logger.debug("ACCEPT: %s", url) url_as_str = urllib.parse.urlunparse(parsed_result) diff --git a/tests/e2e/repo_finder/repo_finder.py b/tests/e2e/repo_finder/repo_finder.py index 90d71acb4..71fea7d68 100644 --- a/tests/e2e/repo_finder/repo_finder.py +++ b/tests/e2e/repo_finder/repo_finder.py @@ -8,7 +8,10 @@ import logging import os -from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder +from packageurl import PackageURL + +from macaron.config.defaults import defaults +from macaron.repo_finder.repo_finder import find_repo logger: logging.Logger = logging.getLogger(__name__) @@ -23,21 +26,31 @@ def test_repo_finder() -> int: - It is extremely unlikely that Maven central will change its API or cease operation in the near future. - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. """ - # Test deps.dev API for a Python package - repo_finder = DepsDevRepoFinder("pypi") - urls = [] - # Without version - urls.append(repo_finder.create_urls("", "packageurl-python", "")) - # With version - urls.append(repo_finder.create_urls("", "packageurl-python", "0.11.1")) - for url in urls: - logger.debug("Testing: %s", url[0]) - metadata = repo_finder.retrieve_metadata(url[0]) - if not metadata: - return os.EX_UNAVAILABLE - links = repo_finder.read_metadata(metadata) - if not links: - return os.EX_UNAVAILABLE + defaults.add_section("repofinder") + defaults.set("repofinder", "use_open_source_insights", "True") + + defaults.add_section("git_service.github") + defaults.set("git_service.github", "domain", "github.com") + + defaults.add_section("git_service.gitlab") + defaults.set("git_service.gitlab", "domain", "gitlab.com") + + # Test deps.dev API for a Python package. + if not find_repo(PackageURL.from_string("pkg:pypi/packageurl-python@0.11.1")): + return os.EX_UNAVAILABLE + + # Test deps.dev API for a Nuget package. + if not find_repo(PackageURL.from_string("pkg:nuget/azure.core")): + return os.EX_UNAVAILABLE + + # Test deps.dev API for an NPM package. + if not find_repo(PackageURL.from_string("pkg:npm/@colors/colors")): + return os.EX_UNAVAILABLE + + # Test deps.dev API for Cargo package. + if not find_repo(PackageURL.from_string("pkg:cargo/rand_core")): + return os.EX_UNAVAILABLE + return os.EX_OK From abcd9a08c2b8a05249af0702fd5f0f460595c980 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Tue, 5 Sep 2023 15:16:11 +1000 Subject: [PATCH 17/31] chore: restored removed test Signed-off-by: Ben Selwyn-Smith --- tests/repo_finder/test_repo_finder.py | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/repo_finder/test_repo_finder.py diff --git a/tests/repo_finder/test_repo_finder.py b/tests/repo_finder/test_repo_finder.py new file mode 100644 index 000000000..4aafe308d --- /dev/null +++ b/tests/repo_finder/test_repo_finder.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the repo finder.""" + +import pytest +from packageurl import PackageURL + +from macaron.config.target_config import Configuration +from macaron.slsa_analyzer.analyzer import Analyzer + + +@pytest.mark.parametrize( + ("config", "available_domains", "expect"), + [ + ( + Configuration({"purl": ""}), + ["github.com", "gitlab.com", "bitbucket.org"], + Analyzer.AnalysisTarget(parsed_purl=None, repo_path="", branch="", digest=""), + ), + ( + Configuration({"purl": "pkg:github.com/apache/maven"}), + ["github.com", "gitlab.com", "bitbucket.org"], + Analyzer.AnalysisTarget( + parsed_purl=PackageURL.from_string("pkg:github.com/apache/maven"), + repo_path="https://github.com/apache/maven", + branch="", + digest="", + ), + ), + ( + Configuration({"purl": "", "path": "https://github.com/apache/maven"}), + ["github.com", "gitlab.com", "bitbucket.org"], + Analyzer.AnalysisTarget( + parsed_purl=None, repo_path="https://github.com/apache/maven", branch="", digest="" + ), + ), + ( + Configuration({"purl": "pkg:maven/apache/maven", "path": "https://github.com/apache/maven"}), + ["github.com", "gitlab.com", "bitbucket.org"], + Analyzer.AnalysisTarget( + parsed_purl=PackageURL.from_string("pkg:maven/apache/maven"), + repo_path="https://github.com/apache/maven", + branch="", + digest="", + ), + ), + ( + Configuration( + { + "purl": "pkg:maven/apache/maven", + "path": "https://github.com/apache/maven", + "branch": "master", + "digest": "abcxyz", + } + ), + ["github.com", "gitlab.com", "bitbucket.org"], + Analyzer.AnalysisTarget( + parsed_purl=PackageURL.from_string("pkg:maven/apache/maven"), + repo_path="https://github.com/apache/maven", + branch="master", + digest="abcxyz", + ), + ), + ], +) +def test_resolve_analysis_target( + config: Configuration, available_domains: list[str], expect: Analyzer.AnalysisTarget +) -> None: + """Test the resolve analysis target method with valid inputs.""" + assert Analyzer.to_analysis_target(config, available_domains) == expect From 011da8b0aaa01f8278c7a7001365d4cda72bcc75 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 6 Sep 2023 11:03:14 +1000 Subject: [PATCH 18/31] chore: repo finder interface refactoring Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 2 +- .../dependency_resolver.py | 4 +- src/macaron/repo_finder/repo_finder.py | 4 +- src/macaron/repo_finder/repo_finder_base.py | 66 ++-------------- .../repo_finder/repo_finder_deps_dev.py | 77 ++++++++----------- src/macaron/repo_finder/repo_finder_java.py | 56 ++++++++------ tests/e2e/repo_finder/repo_finder.py | 29 ++++--- .../repo_finder_deps_dev/__init__.py | 2 - .../test_repo_finder_dd.py | 16 ---- .../repo_finder/repo_finder_java/__init__.py | 2 - .../resources/example_pom.xml | 65 ---------------- .../resources/example_pom_no_scm.xml | 12 --- .../repo_finder_java/test_repo_finder_java.py | 51 ------------ 13 files changed, 96 insertions(+), 290 deletions(-) delete mode 100644 tests/repo_finder/repo_finder_deps_dev/__init__.py delete mode 100644 tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py delete mode 100644 tests/repo_finder/repo_finder_java/__init__.py delete mode 100644 tests/repo_finder/repo_finder_java/resources/example_pom.xml delete mode 100644 tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml delete mode 100644 tests/repo_finder/repo_finder_java/test_repo_finder_java.py diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index f44d588cf..729371974 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -176,7 +176,7 @@ def convert_components_to_artifacts( if component.get("purl"): purl = PackageURL.from_string(str(component.get("purl"))) else: - # TODO remove maven assumption when existence of the component's purl is handled + # TODO remove maven assumption when optional non-existence of the component's purl is handled # See https://github.com/oracle/macaron/issues/464 purl_string = f"pkg:maven/{component.get('group')}/{component.get('name')}" if component.get("version"): diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index 3330cb5cd..cde984ea2 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -164,8 +164,8 @@ def add_latest_version( else: try: if ( - (latest_version := latest_value.get("purl").version) # type: ignore - and (item_version := item.get("purl").version) # type: ignore + (latest_version := latest_value.get("purl").version) # type: ignore[union-attr] + and (item_version := item.get("purl").version) # type: ignore[union-attr] and version.Version(latest_version) < version.Version(item_version) ): latest_deps[key] = item diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 426a4738e..9f874a2f5 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -70,14 +70,14 @@ def find_repo(purl: PackageURL) -> str: "cargo", "npm", ]: - repo_finder = DepsDevRepoFinder(purl.type) + repo_finder = DepsDevRepoFinder() else: logger.debug("No Repo Finder found for package type: %s of %s", purl.type, purl.to_string()) return "" # Call Repo Finder and return first valid URL logger.debug("Analyzing %s with Repo Finder: %s", purl.to_string(), repo_finder.__class__) - urls = repo_finder.find_repo(purl.namespace or "", purl.name, purl.version or "") + urls = repo_finder.find_repo(purl) return find_valid_url(urls) diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py index ee496f53c..ff58354ee 100644 --- a/src/macaron/repo_finder/repo_finder_base.py +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -6,78 +6,24 @@ from abc import ABC, abstractmethod from collections.abc import Iterator +from packageurl import PackageURL + class BaseRepoFinder(ABC): """This abstract class is used to represent Repository Finders.""" @abstractmethod - def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator[str]: """ Attempt to retrieve a repository URL that matches the passed artifact. Parameters ---------- - group : str - The group identifier of an artifact. - artifact : str - The artifact name of an artifact. - version : str - The version number of an artifact. + purl : PackageURL + The PURL of an artifact. Yields ------ Iterator[str] : - The URLs found for the passed GAV. - """ - - @abstractmethod - def create_urls(self, group: str, artifact: str, version: str) -> list[str]: - """ - Create the urls to search for the metadata relating to the passed artifact. - - Parameters - ---------- - group : str - The group ID. - artifact: str - The artifact ID. - version: str - The version of the artifact. - - Returns - ------- - list[str] - The list of created URLs. - """ - - @abstractmethod - def retrieve_metadata(self, url: str) -> str: - """ - Attempt to retrieve the file located at the passed URL. - - Parameters - ---------- - url : str - The URL for the GET request. - - Returns - ------- - str : - The retrieved file data or an empty string. - """ - - @abstractmethod - def read_metadata(self, metadata: str) -> list[str]: - """ - Parse the passed metadata and extract the relevant information. - - Parameters - ---------- - metadata : str - The metadata as a string. - - Returns - ------- - list[str] : - The extracted contents as a list of strings. + The URLs found for the passed artifact. """ diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index eb447d5d8..0a08172c0 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -7,6 +7,7 @@ from collections.abc import Iterator from urllib.parse import quote as encode +from packageurl import PackageURL from requests.exceptions import ReadTimeout from macaron.repo_finder.repo_finder_base import BaseRepoFinder @@ -18,52 +19,38 @@ class DepsDevRepoFinder(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev.""" - def __init__(self, purl_type: str) -> None: - """Initialise the deps.dev repository finder instance. - - Parameters - ---------- - purl_type : str - The PURL type this instance is intended for use with. - """ - self.type = purl_type - - def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator[str]: """ Attempt to retrieve a repository URL that matches the passed artifact. Parameters ---------- - group : str - The group identifier of an artifact. - artifact : str - The artifact name of an artifact. - version : str - The version number of an artifact. + purl : PackageURL + The PURL of an artifact. Yields ------ Iterator[str] : The URLs found for the passed GAV. """ - request_urls = self.create_urls(group, artifact, version) + request_urls = self._create_urls(purl.namespace or "", purl.name, purl.version or "", purl.type) if not request_urls: - logger.debug("No urls found for: %s", artifact) + logger.debug("No urls found for: %s", purl) return - metadata = self.retrieve_metadata(request_urls[0]) - if not metadata: - logger.debug("Failed to retrieve metadata for: %s", artifact) + json_data = self._retrieve_json(request_urls[0]) + if not json_data: + logger.debug("Failed to retrieve json data for: %s", purl) return - urls = self.read_metadata(metadata) + urls = self._read_json(json_data) if not urls: - logger.debug("Failed to extract repository URLs from metadata: %s", artifact) + logger.debug("Failed to extract repository URLs from json data: %s", purl) return yield from iter(urls) - def create_urls(self, group: str, artifact: str, version: str) -> list[str]: + def _create_urls(self, namespace: str, name: str, version: str, type_: str) -> list[str]: """ Create the urls to search for the metadata relating to the passed artifact. @@ -71,19 +58,21 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: Parameters ---------- - group : str - The group ID. - artifact: str - The artifact ID. + namespace : str + The PURL namespace. + name: str + The PURL name. version: str - The version of the artifact. + The PURL version. + type : str + The PURL type. Returns ------- list[str] The list of created URLs. """ - base_url = self.create_type_specific_url(group, artifact) + base_url = self._create_type_specific_url(namespace, name, type_) if not base_url: return [] @@ -95,7 +84,7 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: try: response = send_get_http_raw(base_url, {}) except ReadTimeout: - logger.debug("Failed to retrieve version (timeout): %s:%s", group, artifact) + logger.debug("Failed to retrieve version (timeout): %s:%s", namespace, name) return [] if not response: @@ -111,9 +100,9 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: return [] - def retrieve_metadata(self, url: str) -> str: + def _retrieve_json(self, url: str) -> str: """ - Attempt to retrieve the file located at the passed URL. + Attempt to retrieve the json file located at the passed URL. Parameters ---------- @@ -136,21 +125,21 @@ def retrieve_metadata(self, url: str) -> str: return response.text - def read_metadata(self, metadata: str) -> list[str]: + def _read_json(self, json_data: str) -> list[str]: """ - Parse the deps.dev metadata and extract the repository links. + Parse the deps.dev json file and extract the repository links. Parameters ---------- - metadata : str - The metadata as a string. + json_data : str + The json metadata as a string. Returns ------- list[str] : The extracted contents as a list of strings. """ - parsed = json.loads(metadata) + parsed = json.loads(json_data) if not parsed["links"]: logger.debug("Metadata had no URLs: %s", parsed["versionKey"]) @@ -162,7 +151,7 @@ def read_metadata(self, metadata: str) -> list[str]: return result - def create_type_specific_url(self, namespace: str, name: str) -> str: + def _create_type_specific_url(self, namespace: str, name: str, type_: str) -> str: """Create a URL for the deps.dev API based on the package type. Parameters @@ -171,6 +160,8 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: The PURL namespace element. name : str The PURL name element. + type : str + The PURL type. Returns ------- @@ -181,7 +172,7 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: name = encode(name) # See https://docs.deps.dev/api/v3alpha/ - match self.type: + match type_: case "pypi": package_name = name.lower().replace("_", "-") case "npm": @@ -195,7 +186,7 @@ def create_type_specific_url(self, namespace: str, name: str) -> str: package_name = f"{namespace}%3A{name}" case _: - logger.debug("PURL type not yet supported: %s", self.type) + logger.debug("PURL type not yet supported: %s", type_) return "" - return f"https://api.deps.dev/v3alpha/systems/{self.type}/packages/{package_name}" + return f"https://api.deps.dev/v3alpha/systems/{type_}/packages/{package_name}" diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index e7e0d802d..a9a5af800 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -9,6 +9,7 @@ import defusedxml.ElementTree from defusedxml.ElementTree import fromstring +from packageurl import PackageURL from requests.exceptions import ReadTimeout from macaron.config.defaults import defaults @@ -25,18 +26,14 @@ def __init__(self) -> None: """Initialise the Java repository finder instance.""" self.pom_element: Element | None = None - def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator[str]: """ Attempt to retrieve a repository URL that matches the passed artifact. Parameters ---------- - group : str - The group identifier of an artifact. - artifact : str - The artifact name of an artifact. - version : str - The version number of an artifact. + purl : PackageURL + The PURL of an artifact. Yields ------ @@ -49,12 +46,20 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: # - Try to extract SCM metadata and return URLs # - Try to extract parent information and change current artifact to it # - Repeat - group = group.replace(".", "/") + group = (purl.namespace or "").replace(".", "/") + artifact = purl.name + version = purl.version or "" limit = defaults.getint("repofinder.java", "parent_limit", fallback=10) + + if not version: + logger.debug("Version missing for maven artifact: %s:%s", group, artifact) + # TODO add support for Java artifacts without a version + return + while group and artifact and version and limit > 0: # Create the URLs for retrieving the artifact's POM group = group.replace(".", "/") - request_urls = self.create_urls(group, artifact, version) + request_urls = self._create_urls(group, artifact, version) if not request_urls: # Abort if no URLs were created logger.debug("Failed to create request URLs for %s:%s:%s", group, artifact, version) @@ -63,7 +68,7 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: # Try each POM URL in order, terminating early if a match is found pom = "" for request_url in request_urls: - pom = self.retrieve_metadata(request_url) + pom = self._retrieve_pom(request_url) if pom != "": break @@ -72,15 +77,16 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: logger.debug("No POM found for %s:%s:%s", group, artifact, version) return - urls = self.read_metadata(pom) + urls = self._read_pom(pom) if urls: logger.debug("Found %s urls: %s", len(urls), urls) yield from iter(urls) + break if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: # Attempt to extract parent information from POM - group, artifact, version = self.find_parent(self.pom_element) + group, artifact, version = self._find_parent(self.pom_element) else: break @@ -89,9 +95,9 @@ def find_repo(self, group: str, artifact: str, version: str) -> Iterator[str]: # Nothing found return - def create_urls(self, group: str, artifact: str, version: str) -> list[str]: + def _create_urls(self, group: str, artifact: str, version: str) -> list[str]: """ - Create the urls to search for the metadata relating to the passed artifact. + Create the urls to search for the pom relating to the passed artifact. Parameters ---------- @@ -115,7 +121,7 @@ def create_urls(self, group: str, artifact: str, version: str) -> list[str]: urls.append(f"{repo}/{group}/{artifact}/{version}/{artifact}-{version}.pom") return urls - def retrieve_metadata(self, url: str) -> str: + def _retrieve_pom(self, url: str) -> str: """ Attempt to retrieve the file located at the passed URL. @@ -132,7 +138,7 @@ def retrieve_metadata(self, url: str) -> str: try: response = send_get_http_raw(url, {}) except ReadTimeout: - logger.debug("Failed to retrieve metadata (timeout): %s", url) + logger.debug("Failed to retrieve pom (timeout): %s", url) return "" if not response: @@ -141,14 +147,14 @@ def retrieve_metadata(self, url: str) -> str: logger.debug("Found artifact POM at: %s", url) return response.text - def read_metadata(self, metadata: str) -> list[str]: + def _read_pom(self, pom: str) -> list[str]: """ Parse the passed pom and extract the relevant tags. Parameters ---------- - metadata : str - The metadata as a string. + pom : str + The pom as a string. Returns ------- @@ -162,14 +168,14 @@ def read_metadata(self, metadata: str) -> list[str]: return [] # Parse POM using defusedxml - pom_element = self.parse_pom(metadata) + pom_element = self._parse_pom(pom) if pom_element is None: return [] # Attempt to extract SCM data and return URL - return self.find_scm(pom_element, tags) + return self._find_scm(pom_element, tags) - def parse_pom(self, pom: str) -> Element | None: + def _parse_pom(self, pom: str) -> Element | None: """ Parse the passed POM using defusedxml. @@ -190,7 +196,7 @@ def parse_pom(self, pom: str) -> Element | None: logger.debug("Failed to parse XML: %s", error) return None - def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = True) -> list[str]: + def _find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = True) -> list[str]: """ Parse the passed pom and extract the passed tags. @@ -236,7 +242,7 @@ def find_scm(self, pom: Element, tags: list[str], resolve_properties: bool = Tru return results - def find_parent(self, pom: Element) -> tuple[str, str, str]: + def _find_parent(self, pom: Element) -> tuple[str, str, str]: """ Extract parent information from passed POM. @@ -298,7 +304,7 @@ def _resolve_properties(self, pom: Element, values: list[str]) -> list[str]: else: text = f"properties.{text}" # Call find_scm with property resolution flag set to False to prevent the possibility of endless looping - result = self.find_scm(pom, [text], False) + result = self._find_scm(pom, [text], False) if not result: break replacements.append([match.start(), result[0], match.end()]) diff --git a/tests/e2e/repo_finder/repo_finder.py b/tests/e2e/repo_finder/repo_finder.py index 71fea7d68..0261dbc31 100644 --- a/tests/e2e/repo_finder/repo_finder.py +++ b/tests/e2e/repo_finder/repo_finder.py @@ -20,21 +20,32 @@ def test_repo_finder() -> int: - """Test the functionality of the remote API calls used by the repo finder. - - Functionality relating to Java artifacts is not verified for two reasons: - - It is extremely unlikely that Maven central will change its API or cease operation in the near future. - - Other similar repositories to Maven central (internal Artifactory, etc.) can be provided by the user instead. - """ - defaults.add_section("repofinder") + """Test the functionality of the remote API calls used by the repo finder.""" + if not defaults.has_section("repofinder.java"): + defaults.add_section("repofinder.java") + defaults.set("repofinder.java", "find_parents", "True") + defaults.set("repofinder.java", "repo_pom_paths", "scm.url") + + if not defaults.has_section("repofinder"): + defaults.add_section("repofinder") defaults.set("repofinder", "use_open_source_insights", "True") - defaults.add_section("git_service.github") + if not defaults.has_section("git_service.github"): + defaults.add_section("git_service.github") defaults.set("git_service.github", "domain", "github.com") - defaults.add_section("git_service.gitlab") + if not defaults.has_section("git_service.gitlab"): + defaults.add_section("git_service.gitlab") defaults.set("git_service.gitlab", "domain", "gitlab.com") + # Test Java package with SCM metadata in artifact POM. + if not find_repo(PackageURL.from_string("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.2")): + return os.EX_UNAVAILABLE + + # Test Java package with SCM metadata in artifact's parent POM. + if not find_repo(PackageURL.from_string("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.2")): + return os.EX_UNAVAILABLE + # Test deps.dev API for a Python package. if not find_repo(PackageURL.from_string("pkg:pypi/packageurl-python@0.11.1")): return os.EX_UNAVAILABLE diff --git a/tests/repo_finder/repo_finder_deps_dev/__init__.py b/tests/repo_finder/repo_finder_deps_dev/__init__.py deleted file mode 100644 index 19aeac023..000000000 --- a/tests/repo_finder/repo_finder_deps_dev/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py b/tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py deleted file mode 100644 index 26157e8fa..000000000 --- a/tests/repo_finder/repo_finder_deps_dev/test_repo_finder_dd.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. - -""" -This module tests the python dd repo finder. -""" -from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder - - -def test_repo_finder_deps_dev() -> None: - """Test the functions of the repo finder.""" - repo_finder = DepsDevRepoFinder("pypi") - assert ( - repo_finder.create_type_specific_url("", "packageurl-python") - == "https://api.deps.dev/v3alpha/systems/pypi/packages/packageurl-python" - ) diff --git a/tests/repo_finder/repo_finder_java/__init__.py b/tests/repo_finder/repo_finder_java/__init__.py deleted file mode 100644 index 19aeac023..000000000 --- a/tests/repo_finder/repo_finder_java/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/tests/repo_finder/repo_finder_java/resources/example_pom.xml b/tests/repo_finder/repo_finder_java/resources/example_pom.xml deleted file mode 100644 index 76cf0f35f..000000000 --- a/tests/repo_finder/repo_finder_java/resources/example_pom.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - - - 4.0.0 - - abc.example - example - 0.0.1 - - abc.example.example - example - 0.0.1 - example - - An Example - - https://example.example/example - 2023 - - - Example_License - https://example.example/license - ${licenses.license.distribution} - - - - - ssh://git@hostname:port/owner/${project.licenses.license.name}.git - - - git@github.com:owner/project${javac.src.version}-${project.inceptionYear}.git - - https://github.com/owner/project - example-0.0.1 - - - - 1.8 - 1.8 - - - - e.e.e - f - 0.1 - - - - - release - - true - true - - - - diff --git a/tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml b/tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml deleted file mode 100644 index 8586b1c20..000000000 --- a/tests/repo_finder/repo_finder_java/resources/example_pom_no_scm.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - 4.0.0 - - owner - parent - 1 - - project - diff --git a/tests/repo_finder/repo_finder_java/test_repo_finder_java.py b/tests/repo_finder/repo_finder_java/test_repo_finder_java.py deleted file mode 100644 index e57e35057..000000000 --- a/tests/repo_finder/repo_finder_java/test_repo_finder_java.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. - -""" -This module tests the repo finder. -""" -import os -from pathlib import Path - -from macaron.repo_finder.repo_finder_java import JavaRepoFinder - - -def test_java_repo_finder() -> None: - """Test the functions of the repo finder.""" - group = "group" - artifact = "artifact" - version = "version" - repo_finder = JavaRepoFinder() - created_urls = repo_finder.create_urls(group, artifact, version) - assert created_urls - - resources_dir = Path(__file__).parent.joinpath("resources") - with open(os.path.join(resources_dir, "example_pom.xml"), encoding="utf8") as file: - file_data = file.read() - pom = repo_finder.parse_pom(file_data) - assert pom is not None - found_urls = repo_finder.find_scm( - pom, ["scm.url", "scm.connection", "scm.developerConnection", "licenses.license.distribution"] - ) - assert len(found_urls) == 4 - expected = [ - "https://github.com/owner/project", - "ssh://git@hostname:port/owner/Example_License.git", - "git@github.com:owner/project1.8-2023.git", - "${licenses.license.distribution}", - ] - assert expected == found_urls - - -def test_java_repo_finder_hierarchical() -> None: - """Test the hierarchical capabilities of the repo finder.""" - resources_dir = Path(__file__).parent.joinpath("resources") - repo_finder = JavaRepoFinder() - with open(os.path.join(resources_dir, "example_pom_no_scm.xml"), encoding="utf8") as file: - file_data = file.read() - pom = repo_finder.parse_pom(file_data) - assert pom is not None - group, artifact, version = repo_finder.find_parent(pom) - assert group == "owner" - assert artifact == "parent" - assert version == "1" From 3ab56643afb33f944cc98f1e11009207440dde82 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 6 Sep 2023 14:47:07 +1000 Subject: [PATCH 19/31] chore: updated repo finder return values Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 8 ++++++-- src/macaron/repo_finder/repo_finder_base.py | 8 ++++---- src/macaron/repo_finder/repo_finder_deps_dev.py | 12 ++++++++---- src/macaron/repo_finder/repo_finder_java.py | 15 +++++++++------ tests/e2e/repo_finder/repo_finder.py | 2 +- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 9f874a2f5..15cea599c 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -77,8 +77,12 @@ def find_repo(purl: PackageURL) -> str: # Call Repo Finder and return first valid URL logger.debug("Analyzing %s with Repo Finder: %s", purl.to_string(), repo_finder.__class__) - urls = repo_finder.find_repo(purl) - return find_valid_url(urls) + for urls in repo_finder.find_repo(purl): + url = find_valid_url(urls) + if url: + return url + + return "" def to_domain_from_known_purl_types(purl_type: str) -> str | None: diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py index ff58354ee..1507f0fda 100644 --- a/src/macaron/repo_finder/repo_finder_base.py +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -13,9 +13,9 @@ class BaseRepoFinder(ABC): """This abstract class is used to represent Repository Finders.""" @abstractmethod - def find_repo(self, purl: PackageURL) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator: """ - Attempt to retrieve a repository URL that matches the passed artifact. + Generate iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. Parameters ---------- @@ -24,6 +24,6 @@ def find_repo(self, purl: PackageURL) -> Iterator[str]: Yields ------ - Iterator[str] : - The URLs found for the passed artifact. + Iterator : + An iterator that produces the found URLs. """ diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 0a08172c0..dd6017aec 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -19,9 +19,9 @@ class DepsDevRepoFinder(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev.""" - def find_repo(self, purl: PackageURL) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator[Iterator[str]]: """ - Attempt to retrieve a repository URL that matches the passed artifact. + Return iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. Parameters ---------- @@ -31,8 +31,12 @@ def find_repo(self, purl: PackageURL) -> Iterator[str]: Yields ------ Iterator[str] : - The URLs found for the passed GAV. + The URLs found for the passed artifact. """ + yield from iter(self._find_repo(purl)) # type: ignore[misc] + + def _find_repo(self, purl: PackageURL) -> Iterator[str]: + """Attempt to retrieve a repository URL that matches the passed artifact.""" request_urls = self._create_urls(purl.namespace or "", purl.name, purl.version or "", purl.type) if not request_urls: logger.debug("No urls found for: %s", purl) @@ -48,7 +52,7 @@ def find_repo(self, purl: PackageURL) -> Iterator[str]: logger.debug("Failed to extract repository URLs from json data: %s", purl) return - yield from iter(urls) + yield iter(urls) # type: ignore[misc] def _create_urls(self, namespace: str, name: str, version: str, type_: str) -> list[str]: """ diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index a9a5af800..cbbb04743 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -26,9 +26,9 @@ def __init__(self) -> None: """Initialise the Java repository finder instance.""" self.pom_element: Element | None = None - def find_repo(self, purl: PackageURL) -> Iterator[str]: + def find_repo(self, purl: PackageURL) -> Iterator[Iterator[str]]: """ - Attempt to retrieve a repository URL that matches the passed artifact. + Generate iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. Parameters ---------- @@ -37,9 +37,13 @@ def find_repo(self, purl: PackageURL) -> Iterator[str]: Yields ------ - Iterator[str] : - The URLs found for the passed GAV. + Iterator[Iterator[str]] : + The URLs found for the passed artifact. """ + yield from iter(self._find_repo(purl)) # type: ignore[misc] + + def _find_repo(self, purl: PackageURL) -> Iterator[str]: + """Attempt to retrieve a repository URL that matches the passed artifact.""" # Perform the following in a loop: # - Create URLs for the current artifact POM # - Parse the POM @@ -81,8 +85,7 @@ def find_repo(self, purl: PackageURL) -> Iterator[str]: if urls: logger.debug("Found %s urls: %s", len(urls), urls) - yield from iter(urls) - break + yield iter(urls) # type: ignore[misc] if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: # Attempt to extract parent information from POM diff --git a/tests/e2e/repo_finder/repo_finder.py b/tests/e2e/repo_finder/repo_finder.py index 0261dbc31..3143977bf 100644 --- a/tests/e2e/repo_finder/repo_finder.py +++ b/tests/e2e/repo_finder/repo_finder.py @@ -43,7 +43,7 @@ def test_repo_finder() -> int: return os.EX_UNAVAILABLE # Test Java package with SCM metadata in artifact's parent POM. - if not find_repo(PackageURL.from_string("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.2")): + if not find_repo(PackageURL.from_string("pkg:maven/commons-cli/commons-cli@1.5.0")): return os.EX_UNAVAILABLE # Test deps.dev API for a Python package. From 83be471bf444e1297c9e43000087061e94f27a41 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 6 Sep 2023 15:23:29 +1000 Subject: [PATCH 20/31] chore: correctly added repo finder integration tests; fixed duplicate string replacement in java repo finder Signed-off-by: Ben Selwyn-Smith --- scripts/dev_scripts/integration_tests.sh | 2 +- scripts/dev_scripts/integration_tests_docker.sh | 2 +- src/macaron/repo_finder/repo_finder_java.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index 783c185de..ba4bcc990 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -538,7 +538,7 @@ fi echo -e "\n----------------------------------------------------------------------------------" echo "Testing Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -python $COMPARE_JSON_OUT || log_fail +python $TEST_REPO_FINDER || log_fail if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index 27d18896f..f09a284b7 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -149,7 +149,7 @@ python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail echo -e "\n----------------------------------------------------------------------------------" echo "Test Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" -python $COMPARE_JSON_OUT || log_fail +python $TEST_REPO_FINDER || log_fail if [ $? -ne 0 ]; then echo -e "Expect zero status code but got $?." diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index cbbb04743..18a79b935 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -50,7 +50,7 @@ def _find_repo(self, purl: PackageURL) -> Iterator[str]: # - Try to extract SCM metadata and return URLs # - Try to extract parent information and change current artifact to it # - Repeat - group = (purl.namespace or "").replace(".", "/") + group = purl.namespace or "" artifact = purl.name version = purl.version or "" limit = defaults.getint("repofinder.java", "parent_limit", fallback=10) From 8eae1744c311e785af51b68c911828d3198b0775 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 6 Sep 2023 16:17:54 +1000 Subject: [PATCH 21/31] chore: removed repo finder test in docker integration tests Signed-off-by: Ben Selwyn-Smith --- scripts/dev_scripts/integration_tests.sh | 1 + scripts/dev_scripts/integration_tests_docker.sh | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index ba4bcc990..d348f9fb9 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -535,6 +535,7 @@ then fi # Testing the Repo Finder's remote calls. +# This requires the 'packageurl' Python module echo -e "\n----------------------------------------------------------------------------------" echo "Testing Repo Finder functionality." echo -e "----------------------------------------------------------------------------------\n" diff --git a/scripts/dev_scripts/integration_tests_docker.sh b/scripts/dev_scripts/integration_tests_docker.sh index f09a284b7..20ecd3327 100755 --- a/scripts/dev_scripts/integration_tests_docker.sh +++ b/scripts/dev_scripts/integration_tests_docker.sh @@ -145,17 +145,6 @@ POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/policy_report.js $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.db" || log_fail python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail -# Testing the Repo Finder's remote calls. -echo -e "\n----------------------------------------------------------------------------------" -echo "Test Repo Finder functionality." -echo -e "----------------------------------------------------------------------------------\n" -python $TEST_REPO_FINDER || log_fail -if [ $? -ne 0 ]; -then - echo -e "Expect zero status code but got $?." - log_fail -fi - echo -e "\n----------------------------------------------------------------------------------" echo "Test running the analysis without setting the GITHUB_TOKEN environment variables." echo -e "----------------------------------------------------------------------------------\n" From 8bce8867eb8fddd8eed9c57ec134eeaf09008887 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 7 Sep 2023 12:00:12 +1000 Subject: [PATCH 22/31] chore: moved URL validation to within Repo Finders Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 2 +- src/macaron/repo_finder/repo_finder.py | 36 ++----------------- src/macaron/repo_finder/repo_finder_base.py | 11 +++--- .../repo_finder/repo_finder_deps_dev.py | 33 ++++++++--------- src/macaron/repo_finder/repo_finder_java.py | 29 ++++++++------- src/macaron/util.py | 27 ++++++++++++++ 6 files changed, 66 insertions(+), 72 deletions(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 729371974..93772c963 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -16,7 +16,7 @@ from macaron.dependency_analyzer.dependency_resolver import DependencyAnalyzer, DependencyInfo from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus -from macaron.repo_finder.repo_finder import find_valid_url +from macaron.util import find_valid_url logger: logging.Logger = logging.getLogger(__name__) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 15cea599c..afac5c64d 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -19,7 +19,7 @@ For Python, .NET, Rust, and NodeJS type PURLs, Google's Open Source Insights API is used to find the meta data. In either case, any repository links are extracted from the meta data, then checked for validity via -``DependencyAnalyzer::find_valid_url`` which accepts URLs that point to a Github repository or similar. +``utils::find_valid_url`` which accepts URLs that point to a Github repository or similar. Repository PURLs ---------------- @@ -34,7 +34,6 @@ import logging import os -from collections.abc import Iterable from urllib.parse import ParseResult, urlunparse from packageurl import PackageURL @@ -43,7 +42,6 @@ from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_finder_java import JavaRepoFinder -from macaron.slsa_analyzer.git_url import get_remote_vcs_url logger: logging.Logger = logging.getLogger(__name__) @@ -77,12 +75,7 @@ def find_repo(purl: PackageURL) -> str: # Call Repo Finder and return first valid URL logger.debug("Analyzing %s with Repo Finder: %s", purl.to_string(), repo_finder.__class__) - for urls in repo_finder.find_repo(purl): - url = find_valid_url(urls) - if url: - return url - - return "" + return repo_finder.find_repo(purl) def to_domain_from_known_purl_types(purl_type: str) -> str | None: @@ -155,28 +148,3 @@ def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: fragment="", ) ) - - -def find_valid_url(urls: Iterable[str]) -> str: - """Find a valid URL from the provided URLs. - - Parameters - ---------- - urls : Iterable[str] - An Iterable object containing urls. - - Returns - ------- - str - A valid URL or empty if it can't find any valid URL. - """ - vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} - - # To avoid non-deterministic results we sort the URLs. - vcs_list = sorted(vcs_set) - - if len(vcs_list) < 1: - return "" - - # Report the first valid URL. - return vcs_list.pop() diff --git a/src/macaron/repo_finder/repo_finder_base.py b/src/macaron/repo_finder/repo_finder_base.py index 1507f0fda..ba177c89f 100644 --- a/src/macaron/repo_finder/repo_finder_base.py +++ b/src/macaron/repo_finder/repo_finder_base.py @@ -4,7 +4,6 @@ """This module contains the base class for the repo finders.""" from abc import ABC, abstractmethod -from collections.abc import Iterator from packageurl import PackageURL @@ -13,7 +12,7 @@ class BaseRepoFinder(ABC): """This abstract class is used to represent Repository Finders.""" @abstractmethod - def find_repo(self, purl: PackageURL) -> Iterator: + def find_repo(self, purl: PackageURL) -> str: """ Generate iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. @@ -22,8 +21,8 @@ def find_repo(self, purl: PackageURL) -> Iterator: purl : PackageURL The PURL of an artifact. - Yields - ------ - Iterator : - An iterator that produces the found URLs. + Returns + ------- + str : + The URL of the found repository. """ diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index dd6017aec..34b476b49 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -4,14 +4,13 @@ """This module contains the PythonRepoFinderDD class to be used for finding repositories using deps.dev.""" import json import logging -from collections.abc import Iterator from urllib.parse import quote as encode from packageurl import PackageURL from requests.exceptions import ReadTimeout from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.util import send_get_http_raw +from macaron.util import find_valid_url, send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -19,40 +18,42 @@ class DepsDevRepoFinder(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev.""" - def find_repo(self, purl: PackageURL) -> Iterator[Iterator[str]]: + def find_repo(self, purl: PackageURL) -> str: """ - Return iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. + Attempt to retrieve a repository URL that matches the passed artifact. Parameters ---------- purl : PackageURL The PURL of an artifact. - Yields - ------ - Iterator[str] : - The URLs found for the passed artifact. + Returns + ------- + str : + The URL of the found repository. """ - yield from iter(self._find_repo(purl)) # type: ignore[misc] - - def _find_repo(self, purl: PackageURL) -> Iterator[str]: - """Attempt to retrieve a repository URL that matches the passed artifact.""" request_urls = self._create_urls(purl.namespace or "", purl.name, purl.version or "", purl.type) if not request_urls: logger.debug("No urls found for: %s", purl) - return + return "" json_data = self._retrieve_json(request_urls[0]) if not json_data: logger.debug("Failed to retrieve json data for: %s", purl) - return + return "" urls = self._read_json(json_data) if not urls: logger.debug("Failed to extract repository URLs from json data: %s", purl) - return + return "" + + logger.debug("Found %s urls: %s", len(urls), urls) + url = find_valid_url(urls) + if url: + logger.debug("Found valid url: %s", url) + return url - yield iter(urls) # type: ignore[misc] + return "" def _create_urls(self, namespace: str, name: str, version: str, type_: str) -> list[str]: """ diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 18a79b935..7966e9929 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -4,7 +4,6 @@ """This module contains the JavaRepoFinder class to be used for finding Java repositories.""" import logging import re -from collections.abc import Iterator from xml.etree.ElementTree import Element # nosec import defusedxml.ElementTree @@ -14,7 +13,7 @@ from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.util import send_get_http_raw +from macaron.util import find_valid_url, send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -26,9 +25,9 @@ def __init__(self) -> None: """Initialise the Java repository finder instance.""" self.pom_element: Element | None = None - def find_repo(self, purl: PackageURL) -> Iterator[Iterator[str]]: + def find_repo(self, purl: PackageURL) -> str: """ - Generate iterator from _find_repo that attempts to retrieve a repository URL that matches the passed artifact. + Attempt to retrieve a repository URL that matches the passed artifact. Parameters ---------- @@ -37,13 +36,9 @@ def find_repo(self, purl: PackageURL) -> Iterator[Iterator[str]]: Yields ------ - Iterator[Iterator[str]] : - The URLs found for the passed artifact. + str : + The URL of the found repository. """ - yield from iter(self._find_repo(purl)) # type: ignore[misc] - - def _find_repo(self, purl: PackageURL) -> Iterator[str]: - """Attempt to retrieve a repository URL that matches the passed artifact.""" # Perform the following in a loop: # - Create URLs for the current artifact POM # - Parse the POM @@ -58,7 +53,7 @@ def _find_repo(self, purl: PackageURL) -> Iterator[str]: if not version: logger.debug("Version missing for maven artifact: %s:%s", group, artifact) # TODO add support for Java artifacts without a version - return + return "" while group and artifact and version and limit > 0: # Create the URLs for retrieving the artifact's POM @@ -67,7 +62,7 @@ def _find_repo(self, purl: PackageURL) -> Iterator[str]: if not request_urls: # Abort if no URLs were created logger.debug("Failed to create request URLs for %s:%s:%s", group, artifact, version) - return + return "" # Try each POM URL in order, terminating early if a match is found pom = "" @@ -79,13 +74,17 @@ def _find_repo(self, purl: PackageURL) -> Iterator[str]: if pom == "": # Abort if no POM was found logger.debug("No POM found for %s:%s:%s", group, artifact, version) - return + return "" urls = self._read_pom(pom) if urls: + # If the found URLs fail to validate, finding can continue on to the next parent POM logger.debug("Found %s urls: %s", len(urls), urls) - yield iter(urls) # type: ignore[misc] + url = find_valid_url(urls) + if url: + logger.debug("Found valid url: %s", url) + return url if defaults.getboolean("repofinder.java", "find_parents") and self.pom_element is not None: # Attempt to extract parent information from POM @@ -96,7 +95,7 @@ def _find_repo(self, purl: PackageURL) -> Iterator[str]: limit = limit - 1 # Nothing found - return + return "" def _create_urls(self, group: str, artifact: str, version: str) -> list[str]: """ diff --git a/src/macaron/util.py b/src/macaron/util.py index 0872cf7d2..72ed64253 100644 --- a/src/macaron/util.py +++ b/src/macaron/util.py @@ -8,12 +8,14 @@ import shutil import time import urllib.parse +from collections.abc import Iterable from datetime import datetime import requests from requests.models import Response from macaron.config.defaults import defaults +from macaron.slsa_analyzer.git_url import get_remote_vcs_url logger: logging.Logger = logging.getLogger(__name__) @@ -260,3 +262,28 @@ def get_if_exists(doc: JsonType, path: list[str | int]) -> JsonType | None: else: return None return doc + + +def find_valid_url(urls: Iterable[str]) -> str: + """Find a valid URL from the provided URLs. + + Parameters + ---------- + urls : Iterable[str] + An Iterable object containing urls. + + Returns + ------- + str + A valid URL or empty if it can't find any valid URL. + """ + vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} + + # To avoid non-deterministic results we sort the URLs. + vcs_list = sorted(vcs_set) + + if len(vcs_list) < 1: + return "" + + # Report the first valid URL. + return vcs_list.pop() From 4c8cba69e7bd3f247e905bd584aea487b158a401 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 8 Sep 2023 10:35:10 +1000 Subject: [PATCH 23/31] chore: moved url validator to repo finder Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 4 +-- .../repo_finder/repo_finder_deps_dev.py | 5 ++-- src/macaron/repo_finder/repo_finder_java.py | 5 ++-- src/macaron/util.py | 27 ------------------- 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 93772c963..1db91e5c6 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -16,7 +16,7 @@ from macaron.dependency_analyzer.dependency_resolver import DependencyAnalyzer, DependencyInfo from macaron.errors import MacaronError from macaron.output_reporter.scm import SCMStatus -from macaron.util import find_valid_url +from macaron.repo_finder.repo_validator import find_valid_repository_url logger: logging.Logger = logging.getLogger(__name__) @@ -211,7 +211,7 @@ def convert_components_to_artifacts( ) else: # Find a valid URL. - item["url"] = find_valid_url( + item["url"] = find_valid_repository_url( link.get("url") for link in component.get("externalReferences") # type: ignore ) diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 34b476b49..8aa0f296e 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -10,7 +10,8 @@ from requests.exceptions import ReadTimeout from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.util import find_valid_url, send_get_http_raw +from macaron.repo_finder.repo_validator import find_valid_repository_url +from macaron.util import send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -48,7 +49,7 @@ def find_repo(self, purl: PackageURL) -> str: return "" logger.debug("Found %s urls: %s", len(urls), urls) - url = find_valid_url(urls) + url = find_valid_repository_url(urls) if url: logger.debug("Found valid url: %s", url) return url diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 7966e9929..5df41b967 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -13,7 +13,8 @@ from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder -from macaron.util import find_valid_url, send_get_http_raw +from macaron.repo_finder.repo_validator import find_valid_repository_url +from macaron.util import send_get_http_raw logger: logging.Logger = logging.getLogger(__name__) @@ -81,7 +82,7 @@ def find_repo(self, purl: PackageURL) -> str: if urls: # If the found URLs fail to validate, finding can continue on to the next parent POM logger.debug("Found %s urls: %s", len(urls), urls) - url = find_valid_url(urls) + url = find_valid_repository_url(urls) if url: logger.debug("Found valid url: %s", url) return url diff --git a/src/macaron/util.py b/src/macaron/util.py index 72ed64253..0872cf7d2 100644 --- a/src/macaron/util.py +++ b/src/macaron/util.py @@ -8,14 +8,12 @@ import shutil import time import urllib.parse -from collections.abc import Iterable from datetime import datetime import requests from requests.models import Response from macaron.config.defaults import defaults -from macaron.slsa_analyzer.git_url import get_remote_vcs_url logger: logging.Logger = logging.getLogger(__name__) @@ -262,28 +260,3 @@ def get_if_exists(doc: JsonType, path: list[str | int]) -> JsonType | None: else: return None return doc - - -def find_valid_url(urls: Iterable[str]) -> str: - """Find a valid URL from the provided URLs. - - Parameters - ---------- - urls : Iterable[str] - An Iterable object containing urls. - - Returns - ------- - str - A valid URL or empty if it can't find any valid URL. - """ - vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} - - # To avoid non-deterministic results we sort the URLs. - vcs_list = sorted(vcs_set) - - if len(vcs_list) < 1: - return "" - - # Report the first valid URL. - return vcs_list.pop() From f0ee63626426bec48a6908261089edc04662dba0 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 8 Sep 2023 10:47:09 +1000 Subject: [PATCH 24/31] chore: added repo validator Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_validator.py | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/macaron/repo_finder/repo_validator.py diff --git a/src/macaron/repo_finder/repo_validator.py b/src/macaron/repo_finder/repo_validator.py new file mode 100644 index 000000000..e0ed84207 --- /dev/null +++ b/src/macaron/repo_finder/repo_validator.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module exists to validate URLs in terms of their use as a repository that can be analyzed.""" +from collections.abc import Iterable + +from macaron.slsa_analyzer.git_url import get_remote_vcs_url + + +def find_valid_repository_url(urls: Iterable[str]) -> str: + """Find a valid URL from the provided URLs. + + Parameters + ---------- + urls : Iterable[str] + An Iterable object containing urls. + + Returns + ------- + str + A valid URL or empty if it can't find any valid URL. + """ + vcs_set = {get_remote_vcs_url(value) for value in urls if get_remote_vcs_url(value) != ""} + + # To avoid non-deterministic results we sort the URLs. + vcs_list = sorted(vcs_set) + + if len(vcs_list) < 1: + return "" + + # Report the first valid URL from the end of the list. + return vcs_list.pop() From 50137f7a87d9b75f3c7b46e79ee69ae72b70664f Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Fri, 8 Sep 2023 11:32:06 +1000 Subject: [PATCH 25/31] chore: updated docs Signed-off-by: Ben Selwyn-Smith --- .../apidoc/macaron.dependency_analyzer.rst | 22 ++++++-- .../apidoc/macaron.parsers.rst | 8 +++ .../apidoc/macaron.repo_finder.rst | 50 +++++++++++++++++++ .../pages/developers_guide/apidoc/macaron.rst | 1 + .../macaron.slsa_analyzer.build_tool.rst | 8 +++ 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 docs/source/pages/developers_guide/apidoc/macaron.repo_finder.rst diff --git a/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst b/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst index a8dc9894a..723de69d3 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst @@ -33,6 +33,22 @@ macaron.dependency\_analyzer.cyclonedx\_mvn module :undoc-members: :show-inheritance: +macaron.dependency\_analyzer.cyclonedx\_pip module +-------------------------------------------------- + +.. automodule:: macaron.dependency_analyzer.cyclonedx_pip + :members: + :undoc-members: + :show-inheritance: + +macaron.dependency\_analyzer.cyclonedx\_poetry module +----------------------------------------------------- + +.. automodule:: macaron.dependency_analyzer.cyclonedx_poetry + :members: + :undoc-members: + :show-inheritance: + macaron.dependency\_analyzer.dependency\_resolver module -------------------------------------------------------- @@ -41,10 +57,10 @@ macaron.dependency\_analyzer.dependency\_resolver module :undoc-members: :show-inheritance: -macaron.dependency\_analyzer.java\_repo\_finder module ------------------------------------------------------- +macaron.dependency\_analyzer.pip\_resolver module +------------------------------------------------- -.. automodule:: macaron.dependency_analyzer.java_repo_finder +.. automodule:: macaron.dependency_analyzer.pip_resolver :members: :undoc-members: :show-inheritance: diff --git a/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst b/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst index 59779254d..3708c8ce2 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst @@ -32,3 +32,11 @@ macaron.parsers.bashparser module :members: :undoc-members: :show-inheritance: + +macaron.parsers.limited\_xmlparser module +----------------------------------------- + +.. automodule:: macaron.parsers.limited_xmlparser + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/developers_guide/apidoc/macaron.repo_finder.rst b/docs/source/pages/developers_guide/apidoc/macaron.repo_finder.rst new file mode 100644 index 000000000..9276a5e92 --- /dev/null +++ b/docs/source/pages/developers_guide/apidoc/macaron.repo_finder.rst @@ -0,0 +1,50 @@ +macaron.repo\_finder package +============================ + +.. automodule:: macaron.repo_finder + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +macaron.repo\_finder.repo\_finder module +---------------------------------------- + +.. automodule:: macaron.repo_finder.repo_finder + :members: + :undoc-members: + :show-inheritance: + +macaron.repo\_finder.repo\_finder\_base module +---------------------------------------------- + +.. automodule:: macaron.repo_finder.repo_finder_base + :members: + :undoc-members: + :show-inheritance: + +macaron.repo\_finder.repo\_finder\_deps\_dev module +--------------------------------------------------- + +.. automodule:: macaron.repo_finder.repo_finder_deps_dev + :members: + :undoc-members: + :show-inheritance: + +macaron.repo\_finder.repo\_finder\_java module +---------------------------------------------- + +.. automodule:: macaron.repo_finder.repo_finder_java + :members: + :undoc-members: + :show-inheritance: + +macaron.repo\_finder.repo\_validator module +------------------------------------------- + +.. automodule:: macaron.repo_finder.repo_validator + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pages/developers_guide/apidoc/macaron.rst b/docs/source/pages/developers_guide/apidoc/macaron.rst index 9a95b8fc8..fa9eace38 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.rst @@ -19,6 +19,7 @@ Subpackages macaron.output_reporter macaron.parsers macaron.policy_engine + macaron.repo_finder macaron.slsa_analyzer Submodules diff --git a/docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.build_tool.rst b/docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.build_tool.rst index c29968dc8..0ff9c8ef4 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.build_tool.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.slsa_analyzer.build_tool.rst @@ -17,6 +17,14 @@ macaron.slsa\_analyzer.build\_tool.base\_build\_tool module :undoc-members: :show-inheritance: +macaron.slsa\_analyzer.build\_tool.docker module +------------------------------------------------ + +.. automodule:: macaron.slsa_analyzer.build_tool.docker + :members: + :undoc-members: + :show-inheritance: + macaron.slsa\_analyzer.build\_tool.gradle module ------------------------------------------------ From 9d4f657dea711c39121a67bb00a899bd7a566d5f Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 14 Sep 2023 15:16:42 +1000 Subject: [PATCH 26/31] chore: rebase and integrate with config purl change Signed-off-by: Ben Selwyn-Smith --- src/macaron/output_reporter/reporter.py | 3 + src/macaron/slsa_analyzer/analyzer.py | 50 +- .../__snapshots__/test_cyclonedx.ambr | 2212 ++++++++++++----- .../test_dependency_analyzer.py | 36 +- tests/slsa_analyzer/test_analyzer.py | 3 +- 5 files changed, 1642 insertions(+), 662 deletions(-) diff --git a/src/macaron/output_reporter/reporter.py b/src/macaron/output_reporter/reporter.py index 6ff9b3898..5e6feb61c 100644 --- a/src/macaron/output_reporter/reporter.py +++ b/src/macaron/output_reporter/reporter.py @@ -120,6 +120,9 @@ def generate(self, target_dir: str, report: Report | dict) -> None: try: dep_file_name = os.path.join(target_dir, "dependencies.json") serialized_configs = list(report.get_serialized_configs()) + for report_dict in serialized_configs: + # Serialize PackageURL objects as strings + report_dict["purl"] = str(report_dict.get("purl")) self.write_file(dep_file_name, json.dumps(serialized_configs, indent=self.indent)) for record in report.get_records(): diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index f02c12216..c6fa213b7 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -527,20 +527,31 @@ def to_analysis_target(config: Configuration, available_domains: list[str]) -> A InvalidPURLError If the PURL provided from the user is invalid. """ - # Due to the current design of Configuration class. purl, repo_path, branch and digest are initialized - # as empty strings and we assumed that they are always set with input values as non-empty strings. + # Due to the current design of Configuration class, repo_path, branch and digest are initialized + # as empty strings, and we assumed that they are always set with input values as non-empty strings. # Therefore, their true types are ``str``, and an empty string indicates that the input value is not provided. - purl_input_str: str = config.get_value("purl") + # The purl might be a PackageURL type, or a string, and therefore requires extra care. + parsed_purl: PackageURL | None + if config.get_value("purl") is None or config.get_value("purl") == "": + parsed_purl = None + elif isinstance(config.get_value("purl"), str): + try: + parsed_purl = PackageURL.from_string(config.get_value("purl")) + except InvalidPURLError as error: + raise PURLNotFoundError("Invalid input PURL.") from error + else: + parsed_purl = config.get_value("purl") + repo_path_input: str = config.get_value("path") input_branch: str = config.get_value("branch") input_digest: str = config.get_value("digest") - match (purl_input_str, repo_path_input): - case ("", ""): + match (parsed_purl, repo_path_input): + case (None, ""): return Analyzer.AnalysisTarget(parsed_purl=None, repo_path="", branch="", digest="") - case ("", _): - # If only the repository path is provided, we will use the user-provided repository path to created the + case (None, _): + # If only the repository path is provided, we will use the user-provided repository path to create the # ``Repository`` instance. Note that if this case happen, the software component will be initialized # with the PURL generated from the ``Repository`` instance (i.e. as a PURL pointing to a git repository # at a specific commit). For example: ``pkg:github.com/org/name@``. @@ -549,33 +560,24 @@ def to_analysis_target(config: Configuration, available_domains: list[str]) -> A ) case (_, ""): - # If a PURL but no repository path are provided, we try to extract the repository path from the PURL. + # If a PURL but no repository path is provided, we try to extract the repository path from the PURL. # Note that we can't always extract the repository path from any provided PURL. - try: - parsed_purl = PackageURL.from_string(purl_input_str) - except ValueError as error: - raise InvalidPURLError(f"The package url {purl_input_str} is not valid.") from error - - converted_repo_path = repo_finder.to_repo_path(parsed_purl, available_domains) - # TODO: If ``converted_repo_path`` is None, this means that the PURL given by the user if not pointing - # to a git repository (e.g ``pkg:maven/apache/maven@1.0.0``). Resolving the repository - # path from such PURl string will be handled in https://github.com/oracle/macaron/pull/388. + repo = "" + converted_repo_path = repo_finder.to_repo_path(parsed_purl, available_domains) # type: ignore[arg-type] + if converted_repo_path is None: + # Try to find repo from PURL + repo = repo_finder.find_repo(parsed_purl) # type: ignore[arg-type] return Analyzer.AnalysisTarget( parsed_purl=parsed_purl, - repo_path=converted_repo_path or "", + repo_path=converted_repo_path or repo, branch=input_branch, digest=input_digest, ) case (_, _): # If both the PURL and the repository are provided, we will use the user-provided repository path to - # created the ``Repository`` instance later on. This ``Repository`` instance is attached to the + # create the ``Repository`` instance later on. This ``Repository`` instance is attached to the # software component initialized from the user-provided PURL. - try: - parsed_purl = PackageURL.from_string(purl_input_str) - except ValueError as error: - raise InvalidPURLError(f"The package url {purl_input_str} is not valid.") from error - return Analyzer.AnalysisTarget( parsed_purl=parsed_purl, repo_path=repo_path_input, branch=input_branch, digest=input_digest ) diff --git a/tests/dependency_analyzer/cyclonedx/__snapshots__/test_cyclonedx.ambr b/tests/dependency_analyzer/cyclonedx/__snapshots__/test_cyclonedx.ambr index 4f08850d9..91592a119 100644 --- a/tests/dependency_analyzer/cyclonedx/__snapshots__/test_cyclonedx.ambr +++ b/tests/dependency_analyzer/cyclonedx/__snapshots__/test_cyclonedx.ambr @@ -3,1398 +3,2328 @@ dict({ 'ch.qos.logback.contrib:logback-json-classic': dict({ 'available': , - 'group': 'ch.qos.logback.contrib', - 'name': 'logback-json-classic', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/ch.qos.logback.contrib/logback-json-classic@0.1.5?type=jar', + 'purl': PackageURL( + name='logback-json-classic', + namespace='ch.qos.logback.contrib', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='0.1.5', + ), 'url': '', - 'version': '0.1.5', }), 'ch.qos.logback.contrib:logback-json-core': dict({ 'available': , - 'group': 'ch.qos.logback.contrib', - 'name': 'logback-json-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/ch.qos.logback.contrib/logback-json-core@0.1.5?type=jar', + 'purl': PackageURL( + name='logback-json-core', + namespace='ch.qos.logback.contrib', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='0.1.5', + ), 'url': '', - 'version': '0.1.5', }), 'ch.qos.logback:logback-classic': dict({ 'available': , - 'group': 'ch.qos.logback', - 'name': 'logback-classic', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/ch.qos.logback/logback-classic@1.2.11?type=jar', + 'purl': PackageURL( + name='logback-classic', + namespace='ch.qos.logback', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2.11', + ), 'url': '', - 'version': '1.2.11', }), 'ch.qos.logback:logback-core': dict({ 'available': , - 'group': 'ch.qos.logback', - 'name': 'logback-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/ch.qos.logback/logback-core@1.2.11?type=jar', + 'purl': PackageURL( + name='logback-core', + namespace='ch.qos.logback', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2.11', + ), 'url': '', - 'version': '1.2.11', }), 'com.amazon.alexa:ask-sdk-core': dict({ 'available': , - 'group': 'com.amazon.alexa', - 'name': 'ask-sdk-core', 'note': 'https://github.com/amzn/alexa-skills-kit-java is already analyzed.', - 'purl': 'pkg:maven/com.amazon.alexa/ask-sdk-core@2.49.0?type=jar', + 'purl': PackageURL( + name='ask-sdk-core', + namespace='com.amazon.alexa', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.49.0', + ), 'url': 'https://github.com/amzn/alexa-skills-kit-java', - 'version': '2.49.0', }), 'com.amazon.alexa:ask-sdk-lambda-support': dict({ 'available': , - 'group': 'com.amazon.alexa', - 'name': 'ask-sdk-lambda-support', 'note': '', - 'purl': 'pkg:maven/com.amazon.alexa/ask-sdk-lambda-support@2.49.0?type=jar', + 'purl': PackageURL( + name='ask-sdk-lambda-support', + namespace='com.amazon.alexa', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.49.0', + ), 'url': 'https://github.com/amzn/alexa-skills-kit-java', - 'version': '2.49.0', }), 'com.amazon.alexa:ask-sdk-model': dict({ 'available': , - 'group': 'com.amazon.alexa', - 'name': 'ask-sdk-model', 'note': 'https://github.com/amzn/alexa-skills-kit-java is already analyzed.', - 'purl': 'pkg:maven/com.amazon.alexa/ask-sdk-model@1.43.0?type=jar', + 'purl': PackageURL( + name='ask-sdk-model', + namespace='com.amazon.alexa', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.43.0', + ), 'url': 'https://github.com/amzn/alexa-skills-kit-java', - 'version': '1.43.0', }), 'com.amazon.alexa:ask-sdk-model-runtime': dict({ 'available': , - 'group': 'com.amazon.alexa', - 'name': 'ask-sdk-model-runtime', 'note': 'https://github.com/amzn/alexa-skills-kit-java is already analyzed.', - 'purl': 'pkg:maven/com.amazon.alexa/ask-sdk-model-runtime@1.0.5?type=jar', + 'purl': PackageURL( + name='ask-sdk-model-runtime', + namespace='com.amazon.alexa', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.0.5', + ), 'url': 'https://github.com/amzn/alexa-skills-kit-java', - 'version': '1.0.5', }), 'com.amazon.alexa:ask-sdk-runtime': dict({ 'available': , - 'group': 'com.amazon.alexa', - 'name': 'ask-sdk-runtime', 'note': 'https://github.com/amzn/alexa-skills-kit-java is already analyzed.', - 'purl': 'pkg:maven/com.amazon.alexa/ask-sdk-runtime@2.49.0?type=jar', + 'purl': PackageURL( + name='ask-sdk-runtime', + namespace='com.amazon.alexa', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.49.0', + ), 'url': 'https://github.com/amzn/alexa-skills-kit-java', - 'version': '2.49.0', }), 'com.amazonaws.serverless:aws-serverless-java-container-core': dict({ 'available': , - 'group': 'com.amazonaws.serverless', - 'name': 'aws-serverless-java-container-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.amazonaws.serverless/aws-serverless-java-container-core@1.9.1?type=jar', + 'purl': PackageURL( + name='aws-serverless-java-container-core', + namespace='com.amazonaws.serverless', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.9.1', + ), 'url': '', - 'version': '1.9.1', }), 'com.amazonaws:aws-java-sdk-core': dict({ 'available': , - 'group': 'com.amazonaws', - 'name': 'aws-java-sdk-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.amazonaws/aws-java-sdk-core@1.12.382?type=jar', + 'purl': PackageURL( + name='aws-java-sdk-core', + namespace='com.amazonaws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.12.382', + ), 'url': '', - 'version': '1.12.382', }), 'com.amazonaws:aws-java-sdk-lambda': dict({ 'available': , - 'group': 'com.amazonaws', - 'name': 'aws-java-sdk-lambda', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.amazonaws/aws-java-sdk-lambda@1.12.382?type=jar', + 'purl': PackageURL( + name='aws-java-sdk-lambda', + namespace='com.amazonaws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.12.382', + ), 'url': '', - 'version': '1.12.382', }), 'com.amazonaws:aws-lambda-java-core': dict({ 'available': , - 'group': 'com.amazonaws', - 'name': 'aws-lambda-java-core', 'note': '', - 'purl': 'pkg:maven/com.amazonaws/aws-lambda-java-core@1.2.2?type=jar', + 'purl': PackageURL( + name='aws-lambda-java-core', + namespace='com.amazonaws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2.2', + ), 'url': 'https://github.com/aws/aws-lambda-java-libs', - 'version': '1.2.2', }), 'com.amazonaws:aws-lambda-java-events': dict({ 'available': , - 'group': 'com.amazonaws', - 'name': 'aws-lambda-java-events', 'note': 'https://github.com/aws/aws-lambda-java-libs is already analyzed.', - 'purl': 'pkg:maven/com.amazonaws/aws-lambda-java-events@3.11.0?type=jar', + 'purl': PackageURL( + name='aws-lambda-java-events', + namespace='com.amazonaws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.11.0', + ), 'url': 'https://github.com/aws/aws-lambda-java-libs', - 'version': '3.11.0', }), 'com.amazonaws:jmespath-java': dict({ 'available': , - 'group': 'com.amazonaws', - 'name': 'jmespath-java', 'note': '', - 'purl': 'pkg:maven/com.amazonaws/jmespath-java@1.12.382?type=jar', + 'purl': PackageURL( + name='jmespath-java', + namespace='com.amazonaws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.12.382', + ), 'url': 'https://github.com/aws/aws-sdk-java', - 'version': '1.12.382', }), 'com.fasterxml.jackson.core:jackson-annotations': dict({ 'available': , - 'group': 'com.fasterxml.jackson.core', - 'name': 'jackson-annotations', 'note': '', - 'purl': 'pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-annotations', + namespace='com.fasterxml.jackson.core', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': 'https://github.com/FasterXML/jackson-annotations', - 'version': '2.14.1', }), 'com.fasterxml.jackson.core:jackson-core': dict({ 'available': , - 'group': 'com.fasterxml.jackson.core', - 'name': 'jackson-core', 'note': '', - 'purl': 'pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-core', + namespace='com.fasterxml.jackson.core', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': 'https://github.com/FasterXML/jackson-core', - 'version': '2.14.1', }), 'com.fasterxml.jackson.core:jackson-databind': dict({ 'available': , - 'group': 'com.fasterxml.jackson.core', - 'name': 'jackson-databind', 'note': '', - 'purl': 'pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-databind', + namespace='com.fasterxml.jackson.core', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': 'https://github.com/FasterXML/jackson-databind', - 'version': '2.14.1', }), 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor': dict({ 'available': , - 'group': 'com.fasterxml.jackson.dataformat', - 'name': 'jackson-dataformat-cbor', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.fasterxml.jackson.dataformat/jackson-dataformat-cbor@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-dataformat-cbor', + namespace='com.fasterxml.jackson.dataformat', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': '', - 'version': '2.14.1', }), 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8': dict({ 'available': , - 'group': 'com.fasterxml.jackson.datatype', - 'name': 'jackson-datatype-jdk8', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.fasterxml.jackson.datatype/jackson-datatype-jdk8@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-datatype-jdk8', + namespace='com.fasterxml.jackson.datatype', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': '', - 'version': '2.14.1', }), 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310': dict({ 'available': , - 'group': 'com.fasterxml.jackson.datatype', - 'name': 'jackson-datatype-jsr310', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.fasterxml.jackson.datatype/jackson-datatype-jsr310@2.14.1?type=jar', + 'purl': PackageURL( + name='jackson-datatype-jsr310', + namespace='com.fasterxml.jackson.datatype', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.14.1', + ), 'url': '', - 'version': '2.14.1', }), 'com.fizzed:rocker-runtime': dict({ 'available': , - 'group': 'com.fizzed', - 'name': 'rocker-runtime', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/com.fizzed/rocker-runtime@1.3.0?type=jar', + 'purl': PackageURL( + name='rocker-runtime', + namespace='com.fizzed', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.3.0', + ), 'url': '', - 'version': '1.3.0', }), 'com.googlecode.javaewah:JavaEWAH': dict({ 'available': , - 'group': 'com.googlecode.javaewah', - 'name': 'JavaEWAH', 'note': '', - 'purl': 'pkg:maven/com.googlecode.javaewah/JavaEWAH@1.1.7?type=jar', + 'purl': PackageURL( + name='JavaEWAH', + namespace='com.googlecode.javaewah', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.1.7', + ), 'url': 'https://github.com/lemire/javaewah', - 'version': '1.1.7', }), 'com.typesafe:config': dict({ 'available': , - 'group': 'com.typesafe', - 'name': 'config', 'note': '', - 'purl': 'pkg:maven/com.typesafe/config@1.4.1?type=jar', + 'purl': PackageURL( + name='config', + namespace='com.typesafe', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.4.1', + ), 'url': 'https://github.com/lightbend/config', - 'version': '1.4.1', }), 'commons-codec:commons-codec': dict({ 'available': , - 'group': 'commons-codec', - 'name': 'commons-codec', 'note': '', - 'purl': 'pkg:maven/commons-codec/commons-codec@1.15?type=jar', + 'purl': PackageURL( + name='commons-codec', + namespace='commons-codec', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.15', + ), 'url': 'https://github.com/apache/commons-codec', - 'version': '1.15', }), 'commons-fileupload:commons-fileupload': dict({ 'available': , - 'group': 'commons-fileupload', - 'name': 'commons-fileupload', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/commons-fileupload/commons-fileupload@1.4?type=jar', + 'purl': PackageURL( + name='commons-fileupload', + namespace='commons-fileupload', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.4', + ), 'url': '', - 'version': '1.4', }), 'commons-io:commons-io': dict({ 'available': , - 'group': 'commons-io', - 'name': 'commons-io', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/commons-io/commons-io@2.11.0?type=jar', + 'purl': PackageURL( + name='commons-io', + namespace='commons-io', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.11.0', + ), 'url': '', - 'version': '2.11.0', }), 'commons-logging:commons-logging': dict({ 'available': , - 'group': 'commons-logging', - 'name': 'commons-logging', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/commons-logging/commons-logging@1.2?type=jar', + 'purl': PackageURL( + name='commons-logging', + namespace='commons-logging', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2', + ), 'url': '', - 'version': '1.2', }), 'io.github.java-diff-utils:java-diff-utils': dict({ 'available': , - 'group': 'io.github.java-diff-utils', - 'name': 'java-diff-utils', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.github.java-diff-utils/java-diff-utils@4.10?type=jar', + 'purl': PackageURL( + name='java-diff-utils', + namespace='io.github.java-diff-utils', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.10', + ), 'url': '', - 'version': '4.10', }), 'io.micronaut.aws:aws-alexa': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-alexa', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-alexa@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-alexa', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-alexa-httpserver': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-alexa-httpserver', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-alexa-httpserver@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-alexa-httpserver', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-bom': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-bom', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-bom@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-bom', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-cdk': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-cdk', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-cdk@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-cdk', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-cloudwatch-logging': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-cloudwatch-logging', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-cloudwatch-logging@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-cloudwatch-logging', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-common': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-common', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-common@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-common', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-distributed-configuration': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-distributed-configuration', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-distributed-configuration@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-distributed-configuration', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-parameter-store': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-parameter-store', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-parameter-store@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-parameter-store', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-sdk-v1': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-sdk-v1', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-sdk-v1@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-sdk-v1', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-sdk-v2': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-sdk-v2', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-sdk-v2@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-sdk-v2', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-secretsmanager': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-secretsmanager', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-secretsmanager@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-secretsmanager', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-service-discovery': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-service-discovery', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-service-discovery@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-service-discovery', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:aws-ua': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'aws-ua', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/aws-ua@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='aws-ua', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-aws': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-aws', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-aws@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-aws', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-aws-alexa': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-aws-alexa', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-aws-alexa@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-aws-alexa', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-aws-api-proxy-test': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-aws-api-proxy-test', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-aws-api-proxy-test@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-aws-api-proxy-test', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-aws-custom-runtime': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-aws-custom-runtime', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-aws-custom-runtime@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-aws-custom-runtime', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-aws-test': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-aws-test', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-aws-test@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-aws-test', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:function-client-aws': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'function-client-aws', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/function-client-aws@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='function-client-aws', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:test-suite': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'test-suite', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/test-suite@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='test-suite', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:test-suite-aws-sdk-v2': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'test-suite-aws-sdk-v2', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/test-suite-aws-sdk-v2@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='test-suite-aws-sdk-v2', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:test-suite-groovy': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'test-suite-groovy', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/test-suite-groovy@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='test-suite-groovy', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:test-suite-http-server-tck-function-aws-api-proxy': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'test-suite-http-server-tck-function-aws-api-proxy', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/test-suite-http-server-tck-function-aws-api-proxy@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='test-suite-http-server-tck-function-aws-api-proxy', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.aws:test-suite-kotlin': dict({ 'available': , - 'group': 'io.micronaut.aws', - 'name': 'test-suite-kotlin', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.micronaut.aws/test-suite-kotlin@4.0.0-SNAPSHOT?type=jar', + 'purl': PackageURL( + name='test-suite-kotlin', + namespace='io.micronaut.aws', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.0-SNAPSHOT', + ), 'url': '', - 'version': '4.0.0-SNAPSHOT', }), 'io.micronaut.discovery:micronaut-discovery-client': dict({ 'available': , - 'group': 'io.micronaut.discovery', - 'name': 'micronaut-discovery-client', 'note': '', - 'purl': 'pkg:maven/io.micronaut.discovery/micronaut-discovery-client@3.2.0?type=jar', + 'purl': PackageURL( + name='micronaut-discovery-client', + namespace='io.micronaut.discovery', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.2.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-discovery-client', - 'version': '3.2.0', }), 'io.micronaut.serde:micronaut-serde-api': dict({ 'available': , - 'group': 'io.micronaut.serde', - 'name': 'micronaut-serde-api', 'note': '', - 'purl': 'pkg:maven/io.micronaut.serde/micronaut-serde-api@1.5.0?type=jar', + 'purl': PackageURL( + name='micronaut-serde-api', + namespace='io.micronaut.serde', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.5.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-serialization', - 'version': '1.5.0', }), 'io.micronaut.serde:micronaut-serde-jackson': dict({ 'available': , - 'group': 'io.micronaut.serde', - 'name': 'micronaut-serde-jackson', 'note': 'https://github.com/micronaut-projects/micronaut-serialization is already analyzed.', - 'purl': 'pkg:maven/io.micronaut.serde/micronaut-serde-jackson@1.5.0?type=jar', + 'purl': PackageURL( + name='micronaut-serde-jackson', + namespace='io.micronaut.serde', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.5.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-serialization', - 'version': '1.5.0', }), 'io.micronaut.serde:micronaut-serde-support': dict({ 'available': , - 'group': 'io.micronaut.serde', - 'name': 'micronaut-serde-support', 'note': 'https://github.com/micronaut-projects/micronaut-serialization is already analyzed.', - 'purl': 'pkg:maven/io.micronaut.serde/micronaut-serde-support@1.5.0?type=jar', + 'purl': PackageURL( + name='micronaut-serde-support', + namespace='io.micronaut.serde', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.5.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-serialization', - 'version': '1.5.0', }), 'io.micronaut.starter:micronaut-starter-api': dict({ 'available': , - 'group': 'io.micronaut.starter', - 'name': 'micronaut-starter-api', 'note': '', - 'purl': 'pkg:maven/io.micronaut.starter/micronaut-starter-api@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-starter-api', + namespace='io.micronaut.starter', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-starter', - 'version': '3.8.0', }), 'io.micronaut.starter:micronaut-starter-core': dict({ 'available': , - 'group': 'io.micronaut.starter', - 'name': 'micronaut-starter-core', 'note': 'https://github.com/micronaut-projects/micronaut-starter is already analyzed.', - 'purl': 'pkg:maven/io.micronaut.starter/micronaut-starter-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-starter-core', + namespace='io.micronaut.starter', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-starter', - 'version': '3.8.0', }), 'io.micronaut.test:micronaut-test-core': dict({ 'available': , - 'group': 'io.micronaut.test', - 'name': 'micronaut-test-core', 'note': 'https://github.com/micronaut-projects/micronaut-test is already analyzed.', - 'purl': 'pkg:maven/io.micronaut.test/micronaut-test-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-test-core', + namespace='io.micronaut.test', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-test', - 'version': '3.8.0', }), 'io.micronaut.test:micronaut-test-junit5': dict({ 'available': , - 'group': 'io.micronaut.test', - 'name': 'micronaut-test-junit5', 'note': '', - 'purl': 'pkg:maven/io.micronaut.test/micronaut-test-junit5@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-test-junit5', + namespace='io.micronaut.test', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-test', - 'version': '3.8.0', }), 'io.micronaut.testresources:micronaut-test-resources-build-tools': dict({ 'available': , - 'group': 'io.micronaut.testresources', - 'name': 'micronaut-test-resources-build-tools', 'note': '', - 'purl': 'pkg:maven/io.micronaut.testresources/micronaut-test-resources-build-tools@1.2.3?type=jar', + 'purl': PackageURL( + name='micronaut-test-resources-build-tools', + namespace='io.micronaut.testresources', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2.3', + ), 'url': 'https://github.com/micronaut-projects/micronaut-test-resources', - 'version': '1.2.3', }), 'io.micronaut:micronaut-aop': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-aop', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-aop@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-aop', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-buffer-netty': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-buffer-netty', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-buffer-netty@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-buffer-netty', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-context': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-context', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-context@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-context', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-core': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-core', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-core', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-core-reactive': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-core-reactive', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-core-reactive@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-core-reactive', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-function': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-function', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-function@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-function', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-function-client': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-function-client', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-function-client@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-function-client', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-http': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-http', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-http@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-http', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-http-client': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-http-client', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-http-client@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-http-client', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-http-client-core': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-http-client-core', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-http-client-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-http-client-core', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-http-netty': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-http-netty', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-http-netty@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-http-netty', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-http-server': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-http-server', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-http-server@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-http-server', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-inject': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-inject', 'note': '', - 'purl': 'pkg:maven/io.micronaut/micronaut-inject@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-inject', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-jackson-core': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-jackson-core', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-jackson-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-jackson-core', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-jackson-databind': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-jackson-databind', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-jackson-databind@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-jackson-databind', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-json-core': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-json-core', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-json-core@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-json-core', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-router': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-router', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-router@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-router', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-runtime': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-runtime', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-runtime@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-runtime', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-validation': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-validation', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-validation@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-validation', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.micronaut:micronaut-websocket': dict({ 'available': , - 'group': 'io.micronaut', - 'name': 'micronaut-websocket', 'note': 'https://github.com/micronaut-projects/micronaut-core is already analyzed.', - 'purl': 'pkg:maven/io.micronaut/micronaut-websocket@3.8.0?type=jar', + 'purl': PackageURL( + name='micronaut-websocket', + namespace='io.micronaut', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.8.0', + ), 'url': 'https://github.com/micronaut-projects/micronaut-core', - 'version': '3.8.0', }), 'io.netty:netty-buffer': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-buffer', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-buffer@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-buffer', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-codec': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-codec', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-codec@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-codec', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-codec-http': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-codec-http', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-codec-http@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-codec-http', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-codec-http2': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-codec-http2', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-codec-http2@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-codec-http2', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-codec-socks': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-codec-socks', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-codec-socks@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-codec-socks', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-common': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-common', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-common@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-common', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-handler': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-handler', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-handler@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-handler', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-handler-proxy': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-handler-proxy', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-handler-proxy@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-handler-proxy', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-resolver': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-resolver', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-resolver@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-resolver', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-transport': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-transport', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-transport@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-transport', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-transport-classes-epoll': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-transport-classes-epoll', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-transport-classes-epoll@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-transport-classes-epoll', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.netty:netty-transport-native-unix-common': dict({ 'available': , - 'group': 'io.netty', - 'name': 'netty-transport-native-unix-common', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.netty/netty-transport-native-unix-common@4.1.86.Final?type=jar', + 'purl': PackageURL( + name='netty-transport-native-unix-common', + namespace='io.netty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.1.86.Final', + ), 'url': '', - 'version': '4.1.86.Final', }), 'io.projectreactor:reactor-core': dict({ 'available': , - 'group': 'io.projectreactor', - 'name': 'reactor-core', 'note': '', - 'purl': 'pkg:maven/io.projectreactor/reactor-core@3.5.0?type=jar', + 'purl': PackageURL( + name='reactor-core', + namespace='io.projectreactor', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='3.5.0', + ), 'url': 'https://github.com/reactor/reactor-core', - 'version': '3.5.0', }), 'io.swagger.core.v3:swagger-annotations': dict({ 'available': , - 'group': 'io.swagger.core.v3', - 'name': 'swagger-annotations', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/io.swagger.core.v3/swagger-annotations@2.2.7?type=jar', + 'purl': PackageURL( + name='swagger-annotations', + namespace='io.swagger.core.v3', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.2.7', + ), 'url': '', - 'version': '2.2.7', }), 'jakarta.annotation:jakarta.annotation-api': dict({ 'available': , - 'group': 'jakarta.annotation', - 'name': 'jakarta.annotation-api', 'note': '', - 'purl': 'pkg:maven/jakarta.annotation/jakarta.annotation-api@2.1.1?type=jar', + 'purl': PackageURL( + name='jakarta.annotation-api', + namespace='jakarta.annotation', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.1.1', + ), 'url': 'https://github.com/eclipse-ee4j/common-annotations-api', - 'version': '2.1.1', }), 'jakarta.inject:jakarta.inject-api': dict({ 'available': , - 'group': 'jakarta.inject', - 'name': 'jakarta.inject-api', 'note': '', - 'purl': 'pkg:maven/jakarta.inject/jakarta.inject-api@2.0.1?type=jar', + 'purl': PackageURL( + name='jakarta.inject-api', + namespace='jakarta.inject', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.0.1', + ), 'url': 'https://github.com/eclipse-ee4j/injection-api', - 'version': '2.0.1', }), 'javax.annotation:javax.annotation-api': dict({ 'available': , - 'group': 'javax.annotation', - 'name': 'javax.annotation-api', 'note': '', - 'purl': 'pkg:maven/javax.annotation/javax.annotation-api@1.3.2?type=jar', + 'purl': PackageURL( + name='javax.annotation-api', + namespace='javax.annotation', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.3.2', + ), 'url': 'https://github.com/javaee/javax.annotation', - 'version': '1.3.2', }), 'javax.inject:javax.inject': dict({ 'available': , - 'group': 'javax.inject', - 'name': 'javax.inject', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/javax.inject/javax.inject@1?type=jar', + 'purl': PackageURL( + name='javax.inject', + namespace='javax.inject', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1', + ), 'url': '', - 'version': '1', }), 'javax.servlet:javax.servlet-api': dict({ 'available': , - 'group': 'javax.servlet', - 'name': 'javax.servlet-api', 'note': '', - 'purl': 'pkg:maven/javax.servlet/javax.servlet-api@4.0.1?type=jar', + 'purl': PackageURL( + name='javax.servlet-api', + namespace='javax.servlet', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.0.1', + ), 'url': 'https://github.com/javaee/servlet-spec', - 'version': '4.0.1', }), 'javax.validation:validation-api': dict({ 'available': , - 'group': 'javax.validation', - 'name': 'validation-api', 'note': '', - 'purl': 'pkg:maven/javax.validation/validation-api@2.0.1.Final?type=jar', + 'purl': PackageURL( + name='validation-api', + namespace='javax.validation', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.0.1.Final', + ), 'url': 'https://github.com/beanvalidation/beanvalidation-api', - 'version': '2.0.1.Final', }), 'joda-time:joda-time': dict({ 'available': , - 'group': 'joda-time', - 'name': 'joda-time', 'note': '', - 'purl': 'pkg:maven/joda-time/joda-time@2.8.1?type=jar', + 'purl': PackageURL( + name='joda-time', + namespace='joda-time', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.8.1', + ), 'url': 'https://github.com/JodaOrg/joda-time', - 'version': '2.8.1', }), 'org.apache.commons:commons-compress': dict({ 'available': , - 'group': 'org.apache.commons', - 'name': 'commons-compress', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.commons/commons-compress@1.21?type=jar', + 'purl': PackageURL( + name='commons-compress', + namespace='org.apache.commons', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.21', + ), 'url': '', - 'version': '1.21', }), 'org.apache.httpcomponents:httpclient': dict({ 'available': , - 'group': 'org.apache.httpcomponents', - 'name': 'httpclient', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.httpcomponents/httpclient@4.5.13?type=jar', + 'purl': PackageURL( + name='httpclient', + namespace='org.apache.httpcomponents', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.5.13', + ), 'url': '', - 'version': '4.5.13', }), 'org.apache.httpcomponents:httpcore': dict({ 'available': , - 'group': 'org.apache.httpcomponents', - 'name': 'httpcore', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.httpcomponents/httpcore@4.4.13?type=jar', + 'purl': PackageURL( + name='httpcore', + namespace='org.apache.httpcomponents', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.4.13', + ), 'url': '', - 'version': '4.4.13', }), 'org.apache.httpcomponents:httpmime': dict({ 'available': , - 'group': 'org.apache.httpcomponents', - 'name': 'httpmime', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.httpcomponents/httpmime@4.5.13?type=jar', + 'purl': PackageURL( + name='httpmime', + namespace='org.apache.httpcomponents', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='4.5.13', + ), 'url': '', - 'version': '4.5.13', }), 'org.apache.logging.log4j:log4j-api': dict({ 'available': , - 'group': 'org.apache.logging.log4j', - 'name': 'log4j-api', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.logging.log4j/log4j-api@2.19.0?type=jar', + 'purl': PackageURL( + name='log4j-api', + namespace='org.apache.logging.log4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.0', + ), 'url': '', - 'version': '2.19.0', }), 'org.apache.logging.log4j:log4j-core': dict({ 'available': , - 'group': 'org.apache.logging.log4j', - 'name': 'log4j-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.logging.log4j/log4j-core@2.19.0?type=jar', + 'purl': PackageURL( + name='log4j-core', + namespace='org.apache.logging.log4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.0', + ), 'url': '', - 'version': '2.19.0', }), 'org.apache.logging.log4j:log4j-slf4j-impl': dict({ 'available': , - 'group': 'org.apache.logging.log4j', - 'name': 'log4j-slf4j-impl', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.apache.logging.log4j/log4j-slf4j-impl@2.19.0?type=jar', + 'purl': PackageURL( + name='log4j-slf4j-impl', + namespace='org.apache.logging.log4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.0', + ), 'url': '', - 'version': '2.19.0', }), 'org.eclipse.jetty:jetty-http': dict({ 'available': , - 'group': 'org.eclipse.jetty', - 'name': 'jetty-http', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.eclipse.jetty/jetty-http@9.4.50.v20221201?type=jar', + 'purl': PackageURL( + name='jetty-http', + namespace='org.eclipse.jetty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='9.4.50.v20221201', + ), 'url': '', - 'version': '9.4.50.v20221201', }), 'org.eclipse.jetty:jetty-io': dict({ 'available': , - 'group': 'org.eclipse.jetty', - 'name': 'jetty-io', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.eclipse.jetty/jetty-io@9.4.50.v20221201?type=jar', + 'purl': PackageURL( + name='jetty-io', + namespace='org.eclipse.jetty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='9.4.50.v20221201', + ), 'url': '', - 'version': '9.4.50.v20221201', }), 'org.eclipse.jetty:jetty-server': dict({ 'available': , - 'group': 'org.eclipse.jetty', - 'name': 'jetty-server', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.eclipse.jetty/jetty-server@9.4.50.v20221201?type=jar', + 'purl': PackageURL( + name='jetty-server', + namespace='org.eclipse.jetty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='9.4.50.v20221201', + ), 'url': '', - 'version': '9.4.50.v20221201', }), 'org.eclipse.jetty:jetty-util': dict({ 'available': , - 'group': 'org.eclipse.jetty', - 'name': 'jetty-util', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.eclipse.jetty/jetty-util@9.4.50.v20221201?type=jar', + 'purl': PackageURL( + name='jetty-util', + namespace='org.eclipse.jetty', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='9.4.50.v20221201', + ), 'url': '', - 'version': '9.4.50.v20221201', }), 'org.eclipse.jgit:org.eclipse.jgit': dict({ 'available': , - 'group': 'org.eclipse.jgit', - 'name': 'org.eclipse.jgit', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.eclipse.jgit/org.eclipse.jgit@5.11.1.202105131744-r?type=jar', + 'purl': PackageURL( + name='org.eclipse.jgit', + namespace='org.eclipse.jgit', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='5.11.1.202105131744-r', + ), 'url': '', - 'version': '5.11.1.202105131744-r', }), 'org.jetbrains.kotlin:kotlin-stdlib': dict({ 'available': , - 'group': 'org.jetbrains.kotlin', - 'name': 'kotlin-stdlib', 'note': 'https://github.com/JetBrains/kotlin is already analyzed.', - 'purl': 'pkg:maven/org.jetbrains.kotlin/kotlin-stdlib@1.7.21?type=jar', + 'purl': PackageURL( + name='kotlin-stdlib', + namespace='org.jetbrains.kotlin', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.21', + ), 'url': 'https://github.com/JetBrains/kotlin', - 'version': '1.7.21', }), 'org.jetbrains.kotlin:kotlin-stdlib-common': dict({ 'available': , - 'group': 'org.jetbrains.kotlin', - 'name': 'kotlin-stdlib-common', 'note': 'https://github.com/JetBrains/kotlin is already analyzed.', - 'purl': 'pkg:maven/org.jetbrains.kotlin/kotlin-stdlib-common@1.7.21?type=jar', + 'purl': PackageURL( + name='kotlin-stdlib-common', + namespace='org.jetbrains.kotlin', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.21', + ), 'url': 'https://github.com/JetBrains/kotlin', - 'version': '1.7.21', }), 'org.jetbrains.kotlin:kotlin-stdlib-jdk7': dict({ 'available': , - 'group': 'org.jetbrains.kotlin', - 'name': 'kotlin-stdlib-jdk7', 'note': '', - 'purl': 'pkg:maven/org.jetbrains.kotlin/kotlin-stdlib-jdk7@1.7.21?type=jar', + 'purl': PackageURL( + name='kotlin-stdlib-jdk7', + namespace='org.jetbrains.kotlin', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.21', + ), 'url': 'https://github.com/JetBrains/kotlin', - 'version': '1.7.21', }), 'org.jetbrains.kotlin:kotlin-stdlib-jdk8': dict({ 'available': , - 'group': 'org.jetbrains.kotlin', - 'name': 'kotlin-stdlib-jdk8', 'note': 'https://github.com/JetBrains/kotlin is already analyzed.', - 'purl': 'pkg:maven/org.jetbrains.kotlin/kotlin-stdlib-jdk8@1.7.21?type=jar', + 'purl': PackageURL( + name='kotlin-stdlib-jdk8', + namespace='org.jetbrains.kotlin', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.21', + ), 'url': 'https://github.com/JetBrains/kotlin', - 'version': '1.7.21', }), 'org.jetbrains:annotations': dict({ 'available': , - 'group': 'org.jetbrains', - 'name': 'annotations', 'note': '', - 'purl': 'pkg:maven/org.jetbrains/annotations@19.0.0?type=jar', + 'purl': PackageURL( + name='annotations', + namespace='org.jetbrains', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='19.0.0', + ), 'url': 'https://github.com/JetBrains/java-annotations', - 'version': '19.0.0', }), 'org.junit.jupiter:junit-jupiter-api': dict({ 'available': , - 'group': 'org.junit.jupiter', - 'name': 'junit-jupiter-api', 'note': '', - 'purl': 'pkg:maven/org.junit.jupiter/junit-jupiter-api@5.9.1?type=jar', + 'purl': PackageURL( + name='junit-jupiter-api', + namespace='org.junit.jupiter', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='5.9.1', + ), 'url': 'https://github.com/junit-team/junit5', - 'version': '5.9.1', }), 'org.junit.platform:junit-platform-commons': dict({ 'available': , - 'group': 'org.junit.platform', - 'name': 'junit-platform-commons', 'note': 'https://github.com/junit-team/junit5 is already analyzed.', - 'purl': 'pkg:maven/org.junit.platform/junit-platform-commons@1.9.1?type=jar', + 'purl': PackageURL( + name='junit-platform-commons', + namespace='org.junit.platform', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.9.1', + ), 'url': 'https://github.com/junit-team/junit5', - 'version': '1.9.1', }), 'org.opentest4j:opentest4j': dict({ 'available': , - 'group': 'org.opentest4j', - 'name': 'opentest4j', 'note': '', - 'purl': 'pkg:maven/org.opentest4j/opentest4j@1.2.0?type=jar', + 'purl': PackageURL( + name='opentest4j', + namespace='org.opentest4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.2.0', + ), 'url': 'https://github.com/ota4j-team/opentest4j', - 'version': '1.2.0', }), 'org.reactivestreams:reactive-streams': dict({ 'available': , - 'group': 'org.reactivestreams', - 'name': 'reactive-streams', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.reactivestreams/reactive-streams@1.0.4?type=jar', + 'purl': PackageURL( + name='reactive-streams', + namespace='org.reactivestreams', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.0.4', + ), 'url': '', - 'version': '1.0.4', }), 'org.slf4j:jcl-over-slf4j': dict({ 'available': , - 'group': 'org.slf4j', - 'name': 'jcl-over-slf4j', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.slf4j/jcl-over-slf4j@1.7.36?type=jar', + 'purl': PackageURL( + name='jcl-over-slf4j', + namespace='org.slf4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.36', + ), 'url': '', - 'version': '1.7.36', }), 'org.slf4j:slf4j-api': dict({ 'available': , - 'group': 'org.slf4j', - 'name': 'slf4j-api', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.slf4j/slf4j-api@1.7.36?type=jar', + 'purl': PackageURL( + name='slf4j-api', + namespace='org.slf4j', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.7.36', + ), 'url': '', - 'version': '1.7.36', }), 'org.yaml:snakeyaml': dict({ 'available': , - 'group': 'org.yaml', - 'name': 'snakeyaml', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/org.yaml/snakeyaml@1.33?type=jar', + 'purl': PackageURL( + name='snakeyaml', + namespace='org.yaml', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.33', + ), 'url': '', - 'version': '1.33', }), 'software.amazon.awscdk:aws-cdk-lib': dict({ 'available': , - 'group': 'software.amazon.awscdk', - 'name': 'aws-cdk-lib', 'note': '', - 'purl': 'pkg:maven/software.amazon.awscdk/aws-cdk-lib@2.59.0?type=jar', + 'purl': PackageURL( + name='aws-cdk-lib', + namespace='software.amazon.awscdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.59.0', + ), 'url': 'https://github.com/aws/aws-cdk', - 'version': '2.59.0', }), 'software.amazon.awscdk:cdk-asset-awscli-v1': dict({ 'available': , - 'group': 'software.amazon.awscdk', - 'name': 'cdk-asset-awscli-v1', 'note': '', - 'purl': 'pkg:maven/software.amazon.awscdk/cdk-asset-awscli-v1@2.2.52?type=jar', + 'purl': PackageURL( + name='cdk-asset-awscli-v1', + namespace='software.amazon.awscdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.2.52', + ), 'url': 'https://github.com/cdklabs/awscdk-asset-awscli', - 'version': '2.2.52', }), 'software.amazon.awscdk:cdk-asset-kubectl-v20': dict({ 'available': , - 'group': 'software.amazon.awscdk', - 'name': 'cdk-asset-kubectl-v20', 'note': '', - 'purl': 'pkg:maven/software.amazon.awscdk/cdk-asset-kubectl-v20@2.1.1?type=jar', + 'purl': PackageURL( + name='cdk-asset-kubectl-v20', + namespace='software.amazon.awscdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.1.1', + ), 'url': 'https://github.com/cdklabs/awscdk-asset-kubectl', - 'version': '2.1.1', }), 'software.amazon.awscdk:cdk-asset-node-proxy-agent-v5': dict({ 'available': , - 'group': 'software.amazon.awscdk', - 'name': 'cdk-asset-node-proxy-agent-v5', 'note': '', - 'purl': 'pkg:maven/software.amazon.awscdk/cdk-asset-node-proxy-agent-v5@2.0.42?type=jar', + 'purl': PackageURL( + name='cdk-asset-node-proxy-agent-v5', + namespace='software.amazon.awscdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.0.42', + ), 'url': 'https://github.com/cdklabs/awscdk-asset-node-proxy-agent', - 'version': '2.0.42', }), 'software.amazon.awssdk:annotations': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'annotations', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/annotations@2.19.14?type=jar', + 'purl': PackageURL( + name='annotations', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:apache-client': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'apache-client', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/apache-client@2.19.14?type=jar', + 'purl': PackageURL( + name='apache-client', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:auth': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'auth', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/auth@2.19.14?type=jar', + 'purl': PackageURL( + name='auth', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:aws-core': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'aws-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/aws-core@2.19.14?type=jar', + 'purl': PackageURL( + name='aws-core', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:aws-json-protocol': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'aws-json-protocol', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/aws-json-protocol@2.19.14?type=jar', + 'purl': PackageURL( + name='aws-json-protocol', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:cloudwatchlogs': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'cloudwatchlogs', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/cloudwatchlogs@2.19.14?type=jar', + 'purl': PackageURL( + name='cloudwatchlogs', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:endpoints-spi': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'endpoints-spi', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/endpoints-spi@2.19.14?type=jar', + 'purl': PackageURL( + name='endpoints-spi', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:http-client-spi': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'http-client-spi', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/http-client-spi@2.19.14?type=jar', + 'purl': PackageURL( + name='http-client-spi', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:json-utils': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'json-utils', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/json-utils@2.19.14?type=jar', + 'purl': PackageURL( + name='json-utils', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:metrics-spi': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'metrics-spi', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/metrics-spi@2.19.14?type=jar', + 'purl': PackageURL( + name='metrics-spi', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:netty-nio-client': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'netty-nio-client', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/netty-nio-client@2.19.14?type=jar', + 'purl': PackageURL( + name='netty-nio-client', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:profiles': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'profiles', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/profiles@2.19.14?type=jar', + 'purl': PackageURL( + name='profiles', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:protocol-core': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'protocol-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/protocol-core@2.19.14?type=jar', + 'purl': PackageURL( + name='protocol-core', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:regions': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'regions', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/regions@2.19.14?type=jar', + 'purl': PackageURL( + name='regions', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:sdk-core': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'sdk-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/sdk-core@2.19.14?type=jar', + 'purl': PackageURL( + name='sdk-core', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:secretsmanager': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'secretsmanager', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/secretsmanager@2.19.14?type=jar', + 'purl': PackageURL( + name='secretsmanager', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:servicediscovery': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'servicediscovery', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/servicediscovery@2.19.14?type=jar', + 'purl': PackageURL( + name='servicediscovery', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:ssm': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'ssm', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/ssm@2.19.14?type=jar', + 'purl': PackageURL( + name='ssm', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:third-party-jackson-core': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'third-party-jackson-core', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/third-party-jackson-core@2.19.14?type=jar', + 'purl': PackageURL( + name='third-party-jackson-core', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.awssdk:utils': dict({ 'available': , - 'group': 'software.amazon.awssdk', - 'name': 'utils', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.awssdk/utils@2.19.14?type=jar', + 'purl': PackageURL( + name='utils', + namespace='software.amazon.awssdk', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.19.14', + ), 'url': '', - 'version': '2.19.14', }), 'software.amazon.eventstream:eventstream': dict({ 'available': , - 'group': 'software.amazon.eventstream', - 'name': 'eventstream', 'note': '', - 'purl': 'pkg:maven/software.amazon.eventstream/eventstream@1.0.1?type=jar', + 'purl': PackageURL( + name='eventstream', + namespace='software.amazon.eventstream', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.0.1', + ), 'url': 'https://github.com/awslabs/aws-eventstream-java', - 'version': '1.0.1', }), 'software.amazon.ion:ion-java': dict({ 'available': , - 'group': 'software.amazon.ion', - 'name': 'ion-java', 'note': 'Manual configuration required. Could not find SCM URL.', - 'purl': 'pkg:maven/software.amazon.ion/ion-java@1.0.2?type=jar', + 'purl': PackageURL( + name='ion-java', + namespace='software.amazon.ion', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.0.2', + ), 'url': '', - 'version': '1.0.2', }), 'software.amazon.jsii:jsii-runtime': dict({ 'available': , - 'group': 'software.amazon.jsii', - 'name': 'jsii-runtime', 'note': '', - 'purl': 'pkg:maven/software.amazon.jsii/jsii-runtime@1.73.0?type=jar', + 'purl': PackageURL( + name='jsii-runtime', + namespace='software.amazon.jsii', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='1.73.0', + ), 'url': 'https://github.com/aws/jsii', - 'version': '1.73.0', }), 'software.constructs:constructs': dict({ 'available': , - 'group': 'software.constructs', - 'name': 'constructs', 'note': '', - 'purl': 'pkg:maven/software.constructs/constructs@10.1.232?type=jar', + 'purl': PackageURL( + name='constructs', + namespace='software.constructs', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='10.1.232', + ), 'url': 'https://github.com/aws/constructs', - 'version': '10.1.232', }), }) # --- @@ -12377,12 +13307,18 @@ dict({ 'None:joda-time': dict({ 'available': , - 'group': '', - 'name': 'joda-time', 'note': '', - 'purl': 'pkg:maven/joda-time/joda-time@2.6?type=jar', + 'purl': PackageURL( + name='joda-time', + namespace='joda-time', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.6', + ), 'url': 'https://github.com/JodaOrg/joda-time', - 'version': '2.6', }), }) # --- @@ -12390,12 +13326,18 @@ dict({ 'joda-time:joda-time': dict({ 'available': , - 'group': 'joda-time', - 'name': 'joda-time', 'note': '', - 'purl': 'pkg:maven/joda-time/joda-time@2.6?type=jar', + 'purl': PackageURL( + name='joda-time', + namespace='joda-time', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.6', + ), 'url': 'https://github.com/JodaOrg/joda-time', - 'version': '', }), }) # --- @@ -12403,12 +13345,18 @@ dict({ 'joda-time:joda-time': dict({ 'available': , - 'group': 'joda-time', - 'name': 'joda-time', 'note': '', - 'purl': 'pkg:maven/joda-time/joda-time@2.6?type=jar', + 'purl': PackageURL( + name='joda-time', + namespace='joda-time', + qualifiers=dict({ + 'type': 'jar', + }), + subpath=None, + type='maven', + version='2.6', + ), 'url': 'https://github.com/JodaOrg/joda-time', - 'version': '2.7', }), }) # --- diff --git a/tests/dependency_analyzer/test_dependency_analyzer.py b/tests/dependency_analyzer/test_dependency_analyzer.py index 873abdf17..9f21f142b 100644 --- a/tests/dependency_analyzer/test_dependency_analyzer.py +++ b/tests/dependency_analyzer/test_dependency_analyzer.py @@ -45,7 +45,14 @@ def test_merge_config(self) -> None: expected_result_no_deps = [ { "id": "com.fasterxml.jackson.core:jackson-annotations", - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle", + "purl": PackageURL( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-annotations", + version="2.14.0-SNAPSHOT", + qualifiers={"type": "bundle"}, + subpath=None, + ), "path": "https://github.com/FasterXML/jackson-annotations", "branch": "", "digest": "", @@ -54,7 +61,14 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-core", - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle", + "purl": PackageURL( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-core", + version="2.14.0-SNAPSHOT", + qualifiers={"type": "bundle"}, + subpath=None, + ), "path": "https://github.com/FasterXML/jackson-core", "branch": "", "digest": "", @@ -84,7 +98,14 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-annotations", - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle", + "purl": PackageURL( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-annotations", + version="2.14.0-SNAPSHOT", + qualifiers={"type": "bundle"}, + subpath=None, + ), "path": "https://github.com/FasterXML/jackson-annotations", "branch": "", "digest": "", @@ -93,7 +114,14 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-core", - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle", + "purl": PackageURL( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-core", + version="2.14.0-SNAPSHOT", + qualifiers={"type": "bundle"}, + subpath=None, + ), "path": "https://github.com/FasterXML/jackson-core", "branch": "", "digest": "", diff --git a/tests/slsa_analyzer/test_analyzer.py b/tests/slsa_analyzer/test_analyzer.py index 1ae7d9194..33c59bebe 100644 --- a/tests/slsa_analyzer/test_analyzer.py +++ b/tests/slsa_analyzer/test_analyzer.py @@ -9,7 +9,6 @@ from packageurl import PackageURL from macaron.config.target_config import Configuration -from macaron.errors import InvalidPURLError from macaron.slsa_analyzer.analyzer import Analyzer from ..macaron_testcase import MacaronTestCase @@ -112,5 +111,5 @@ def test_resolve_analysis_target( ) def test_resolve_analysis_target_invalid_purl(config: Configuration) -> None: """Test the resolve analysis target method with invalid inputs.""" - with pytest.raises(InvalidPURLError): + with pytest.raises(ValueError): # noqa: PT011 Analyzer.to_analysis_target(config, []) From b369fe6cfb90f727fab778eccc0caeb5e3ac9413 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 18 Sep 2023 14:42:12 +1000 Subject: [PATCH 27/31] chore: addressed review feedback Signed-off-by: Ben Selwyn-Smith --- src/macaron/slsa_analyzer/analyzer.py | 13 +++++++------ tests/slsa_analyzer/test_analyzer.py | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index c6fa213b7..6033aea37 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -530,17 +530,18 @@ def to_analysis_target(config: Configuration, available_domains: list[str]) -> A # Due to the current design of Configuration class, repo_path, branch and digest are initialized # as empty strings, and we assumed that they are always set with input values as non-empty strings. # Therefore, their true types are ``str``, and an empty string indicates that the input value is not provided. - # The purl might be a PackageURL type, or a string, and therefore requires extra care. + # The purl might be a PackageURL type, a string, or None, which should be reduced down to an optional + # PackageURL type. parsed_purl: PackageURL | None if config.get_value("purl") is None or config.get_value("purl") == "": parsed_purl = None - elif isinstance(config.get_value("purl"), str): + elif isinstance(config.get_value("purl"), PackageURL): + parsed_purl = config.get_value("purl") + else: try: parsed_purl = PackageURL.from_string(config.get_value("purl")) - except InvalidPURLError as error: - raise PURLNotFoundError("Invalid input PURL.") from error - else: - parsed_purl = config.get_value("purl") + except ValueError as error: + raise InvalidPURLError(f"Invalid input PURL: {config.get_value('purl')}") from error repo_path_input: str = config.get_value("path") input_branch: str = config.get_value("branch") diff --git a/tests/slsa_analyzer/test_analyzer.py b/tests/slsa_analyzer/test_analyzer.py index 33c59bebe..1ae7d9194 100644 --- a/tests/slsa_analyzer/test_analyzer.py +++ b/tests/slsa_analyzer/test_analyzer.py @@ -9,6 +9,7 @@ from packageurl import PackageURL from macaron.config.target_config import Configuration +from macaron.errors import InvalidPURLError from macaron.slsa_analyzer.analyzer import Analyzer from ..macaron_testcase import MacaronTestCase @@ -111,5 +112,5 @@ def test_resolve_analysis_target( ) def test_resolve_analysis_target_invalid_purl(config: Configuration) -> None: """Test the resolve analysis target method with invalid inputs.""" - with pytest.raises(ValueError): # noqa: PT011 + with pytest.raises(InvalidPURLError): Analyzer.to_analysis_target(config, []) From 11ab3aa2393c1f0f95b1f88018a0836e390e2b3c Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 21 Sep 2023 11:33:11 +1000 Subject: [PATCH 28/31] chore: addressed review feedback Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/cyclonedx.py | 10 ++-- .../dependency_resolver.py | 11 +++-- src/macaron/output_reporter/reporter.py | 3 -- src/macaron/repo_finder/repo_finder.py | 2 +- src/macaron/slsa_analyzer/analyzer.py | 12 +++-- .../test_dependency_analyzer.py | 46 ++++--------------- 6 files changed, 33 insertions(+), 51 deletions(-) diff --git a/src/macaron/dependency_analyzer/cyclonedx.py b/src/macaron/dependency_analyzer/cyclonedx.py index 1db91e5c6..d3ff258ff 100644 --- a/src/macaron/dependency_analyzer/cyclonedx.py +++ b/src/macaron/dependency_analyzer/cyclonedx.py @@ -178,10 +178,12 @@ def convert_components_to_artifacts( else: # TODO remove maven assumption when optional non-existence of the component's purl is handled # See https://github.com/oracle/macaron/issues/464 - purl_string = f"pkg:maven/{component.get('group')}/{component.get('name')}" - if component.get("version"): - purl_string = f"{purl_string}@{component.get('version')}" - purl = PackageURL.from_string(purl_string) + purl = PackageURL( + type="maven", + namespace=component.get("group"), + name=component.get("name"), + version=component.get("version") or None, + ) # According to PEP-0589 all keys must be present in a TypedDict. # See https://peps.python.org/pep-0589/#totality diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index cde984ea2..73eeca565 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -163,9 +163,14 @@ def add_latest_version( latest_deps[key] = item else: try: + # These are stored as variables so mypy does not complain about None values (union-attr) + latest_value_purl = latest_value.get("purl") + item_purl = item.get("purl") if ( - (latest_version := latest_value.get("purl").version) # type: ignore[union-attr] - and (item_version := item.get("purl").version) # type: ignore[union-attr] + latest_value_purl is not None + and item_purl is not None + and (latest_version := latest_value_purl.version) + and (item_version := item_purl.version) and version.Version(latest_version) < version.Version(item_version) ): latest_deps[key] = item @@ -216,7 +221,7 @@ def merge_configs( Configuration( { "id": key, - "purl": value.get("purl"), + "purl": str(value.get("purl")), "path": value.get("url"), "branch": "", "digest": "", diff --git a/src/macaron/output_reporter/reporter.py b/src/macaron/output_reporter/reporter.py index 5e6feb61c..6ff9b3898 100644 --- a/src/macaron/output_reporter/reporter.py +++ b/src/macaron/output_reporter/reporter.py @@ -120,9 +120,6 @@ def generate(self, target_dir: str, report: Report | dict) -> None: try: dep_file_name = os.path.join(target_dir, "dependencies.json") serialized_configs = list(report.get_serialized_configs()) - for report_dict in serialized_configs: - # Serialize PackageURL objects as strings - report_dict["purl"] = str(report_dict.get("purl")) self.write_file(dep_file_name, json.dumps(serialized_configs, indent=self.indent)) for record in report.get_records(): diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index afac5c64d..b9b2f97f8 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -19,7 +19,7 @@ For Python, .NET, Rust, and NodeJS type PURLs, Google's Open Source Insights API is used to find the meta data. In either case, any repository links are extracted from the meta data, then checked for validity via -``utils::find_valid_url`` which accepts URLs that point to a Github repository or similar. +``repo_validator::find_valid_repository_url`` which accepts URLs that point to a Github repository or similar. Repository PURLs ---------------- diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 6033aea37..f7002a14f 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -564,10 +564,14 @@ def to_analysis_target(config: Configuration, available_domains: list[str]) -> A # If a PURL but no repository path is provided, we try to extract the repository path from the PURL. # Note that we can't always extract the repository path from any provided PURL. repo = "" - converted_repo_path = repo_finder.to_repo_path(parsed_purl, available_domains) # type: ignore[arg-type] - if converted_repo_path is None: - # Try to find repo from PURL - repo = repo_finder.find_repo(parsed_purl) # type: ignore[arg-type] + converted_repo_path = None + # parsed_purl cannot be None here, but mypy cannot detect that without some extra help. + if parsed_purl is not None: + converted_repo_path = repo_finder.to_repo_path(parsed_purl, available_domains) + if converted_repo_path is None: + # Try to find repo from PURL + repo = repo_finder.find_repo(parsed_purl) + return Analyzer.AnalysisTarget( parsed_purl=parsed_purl, repo_path=converted_repo_path or repo, diff --git a/tests/dependency_analyzer/test_dependency_analyzer.py b/tests/dependency_analyzer/test_dependency_analyzer.py index 9f21f142b..4edd0efc9 100644 --- a/tests/dependency_analyzer/test_dependency_analyzer.py +++ b/tests/dependency_analyzer/test_dependency_analyzer.py @@ -24,17 +24,19 @@ class TestDependencyAnalyzer(MacaronTestCase): def test_merge_config(self) -> None: """Test merging the manual and automatically resolved configurations.""" # Mock automatically resolved dependencies. - purl_string_1 = "pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle" - purl_string_2 = "pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle" + jackson_annotations_purl = ( + "pkg:maven/com.fasterxml.jackson.core/jackson-annotations@2.14.0-SNAPSHOT?type=bundle" + ) + jackson_core_purl = "pkg:maven/com.fasterxml.jackson.core/jackson-core@2.14.0-SNAPSHOT?type=bundle" auto_deps = { "com.fasterxml.jackson.core:jackson-annotations": DependencyInfo( - purl=PackageURL.from_string(purl_string_1), + purl=PackageURL.from_string(jackson_annotations_purl), url="https://github.com/FasterXML/jackson-annotations", note="", available=SCMStatus.AVAILABLE, ), "com.fasterxml.jackson.core:jackson-core": DependencyInfo( - purl=PackageURL.from_string(purl_string_2), + purl=PackageURL.from_string(jackson_core_purl), url="https://github.com/FasterXML/jackson-core", note="", available=SCMStatus.AVAILABLE, @@ -45,14 +47,7 @@ def test_merge_config(self) -> None: expected_result_no_deps = [ { "id": "com.fasterxml.jackson.core:jackson-annotations", - "purl": PackageURL( - type="maven", - namespace="com.fasterxml.jackson.core", - name="jackson-annotations", - version="2.14.0-SNAPSHOT", - qualifiers={"type": "bundle"}, - subpath=None, - ), + "purl": jackson_annotations_purl, "path": "https://github.com/FasterXML/jackson-annotations", "branch": "", "digest": "", @@ -61,14 +56,7 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-core", - "purl": PackageURL( - type="maven", - namespace="com.fasterxml.jackson.core", - name="jackson-core", - version="2.14.0-SNAPSHOT", - qualifiers={"type": "bundle"}, - subpath=None, - ), + "purl": jackson_core_purl, "path": "https://github.com/FasterXML/jackson-core", "branch": "", "digest": "", @@ -98,14 +86,7 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-annotations", - "purl": PackageURL( - type="maven", - namespace="com.fasterxml.jackson.core", - name="jackson-annotations", - version="2.14.0-SNAPSHOT", - qualifiers={"type": "bundle"}, - subpath=None, - ), + "purl": jackson_annotations_purl, "path": "https://github.com/FasterXML/jackson-annotations", "branch": "", "digest": "", @@ -114,14 +95,7 @@ def test_merge_config(self) -> None: }, { "id": "com.fasterxml.jackson.core:jackson-core", - "purl": PackageURL( - type="maven", - namespace="com.fasterxml.jackson.core", - name="jackson-core", - version="2.14.0-SNAPSHOT", - qualifiers={"type": "bundle"}, - subpath=None, - ), + "purl": jackson_core_purl, "path": "https://github.com/FasterXML/jackson-core", "branch": "", "digest": "", From 5b600bf3f3f8d1ea888dbe58351b26d339ecea1c Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 21 Sep 2023 11:40:15 +1000 Subject: [PATCH 29/31] chore: rebase and make use of updated send_get_http_raw Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder_deps_dev.py | 13 ++----------- src/macaron/repo_finder/repo_finder_java.py | 7 +------ 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 8aa0f296e..22d5d039e 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -7,7 +7,6 @@ from urllib.parse import quote as encode from packageurl import PackageURL -from requests.exceptions import ReadTimeout from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_validator import find_valid_repository_url @@ -87,11 +86,7 @@ def _create_urls(self, namespace: str, name: str, version: str, type_: str) -> l return [f"{base_url}/versions/{version}"] # Find the latest version. - try: - response = send_get_http_raw(base_url, {}) - except ReadTimeout: - logger.debug("Failed to retrieve version (timeout): %s:%s", namespace, name) - return [] + response = send_get_http_raw(base_url, {}) if not response: return [] @@ -120,11 +115,7 @@ def _retrieve_json(self, url: str) -> str: str : The retrieved file data or an empty string. """ - try: - response = send_get_http_raw(url, {}) - except ReadTimeout: - logger.debug("Failed to retrieve metadata (timeout): %s", url) - return "" + response = send_get_http_raw(url, {}) if not response: return "" diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 5df41b967..3c9a89daf 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -9,7 +9,6 @@ import defusedxml.ElementTree from defusedxml.ElementTree import fromstring from packageurl import PackageURL -from requests.exceptions import ReadTimeout from macaron.config.defaults import defaults from macaron.repo_finder.repo_finder_base import BaseRepoFinder @@ -138,11 +137,7 @@ def _retrieve_pom(self, url: str) -> str: str : The retrieved file data or an empty string. """ - try: - response = send_get_http_raw(url, {}) - except ReadTimeout: - logger.debug("Failed to retrieve pom (timeout): %s", url) - return "" + response = send_get_http_raw(url, {}) if not response: return "" From bca62d5ec750e72389f7f1df34c3546b665a08a2 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 21 Sep 2023 12:01:36 +1000 Subject: [PATCH 30/31] chore: enable repo finder for sboms Signed-off-by: Ben Selwyn-Smith --- src/macaron/dependency_analyzer/dependency_resolver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/macaron/dependency_analyzer/dependency_resolver.py b/src/macaron/dependency_analyzer/dependency_resolver.py index 73eeca565..de28e04b3 100644 --- a/src/macaron/dependency_analyzer/dependency_resolver.py +++ b/src/macaron/dependency_analyzer/dependency_resolver.py @@ -283,15 +283,21 @@ def resolve_dependencies(main_ctx: Any, sbom_path: str) -> dict[str, DependencyI dict[str, DependencyInfo] A dictionary where artifacts are grouped based on ``artifactId:groupId``. """ + deps_resolved: dict[str, DependencyInfo] = {} + if sbom_path: logger.info("Getting the dependencies from the SBOM defined at %s.", sbom_path) # Import here to avoid circular dependency # pylint: disable=import-outside-toplevel, cyclic-import from macaron.dependency_analyzer.cyclonedx import get_deps_from_sbom - return get_deps_from_sbom(sbom_path) + deps_resolved = get_deps_from_sbom(sbom_path) - deps_resolved: dict[str, DependencyInfo] = {} + # Use repo finder to find more repositories to analyze. + if defaults.getboolean("repofinder", "find_repos"): + DependencyAnalyzer._resolve_more_dependencies(deps_resolved) + + return deps_resolved build_tools = main_ctx.dynamic_data["build_spec"]["tools"] if not build_tools: From 056b3f3bc396401afcba0708c788da81eb1ace31 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 21 Sep 2023 16:01:13 +1000 Subject: [PATCH 31/31] chore: updated docs Signed-off-by: Ben Selwyn-Smith --- .../apidoc/macaron.dependency_analyzer.rst | 24 ------------ .../apidoc/macaron.parsers.rst | 8 ---- docs/source/pages/using.rst | 37 ++++++++++++++++++- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst b/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst index 723de69d3..8b53dbaf9 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst @@ -33,22 +33,6 @@ macaron.dependency\_analyzer.cyclonedx\_mvn module :undoc-members: :show-inheritance: -macaron.dependency\_analyzer.cyclonedx\_pip module --------------------------------------------------- - -.. automodule:: macaron.dependency_analyzer.cyclonedx_pip - :members: - :undoc-members: - :show-inheritance: - -macaron.dependency\_analyzer.cyclonedx\_poetry module ------------------------------------------------------ - -.. automodule:: macaron.dependency_analyzer.cyclonedx_poetry - :members: - :undoc-members: - :show-inheritance: - macaron.dependency\_analyzer.dependency\_resolver module -------------------------------------------------------- @@ -56,11 +40,3 @@ macaron.dependency\_analyzer.dependency\_resolver module :members: :undoc-members: :show-inheritance: - -macaron.dependency\_analyzer.pip\_resolver module -------------------------------------------------- - -.. automodule:: macaron.dependency_analyzer.pip_resolver - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst b/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst index 3708c8ce2..59779254d 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.parsers.rst @@ -32,11 +32,3 @@ macaron.parsers.bashparser module :members: :undoc-members: :show-inheritance: - -macaron.parsers.limited\_xmlparser module ------------------------------------------ - -.. automodule:: macaron.parsers.limited_xmlparser - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/pages/using.rst b/docs/source/pages/using.rst index 95a0e2352..aebc0e0bc 100644 --- a/docs/source/pages/using.rst +++ b/docs/source/pages/using.rst @@ -104,7 +104,7 @@ To simplify the examples, we use the same configurations as above if needed (e.g The list bellow shows examples for the corresponding PURL strings for different git repositories: -.. list-table:: Example of PURL strings for git repositories. +.. list-table:: Examples of PURL strings for git repositories. :widths: 50 50 :header-rows: 1 @@ -133,6 +133,39 @@ You can also provide the PURL string together with the repository path. In this .. note:: When providing the PURL and the repository path, both the branch name and commit digest must be provided as well. +'''''''''''''''''''''''''''''''''''''' +Providing an artifact as a PURL string +'''''''''''''''''''''''''''''''''''''' + +The PURL format supports artifacts as well as repositories, and Macaron supports (some of) these too. + +.. code-block:: + + pkg:/ + +Where ``artifact_details`` varies based on the provided ``package_type``. Examples for those currently supported by Macaron are as follows: + +.. list-table:: Examples of PURL strings for artifacts. + :widths: 50 50 + :header-rows: 1 + + * - Package Type + - PURL String + * - Maven (Java) + - ``pkg:maven/org.apache.xmlgraphics/batik-anim@1.9.1`` + * - PyPi (Python) + - ``pkg:pypi/django@1.11.1`` + * - Cargo (Rust) + - ``pkg:cargo/rand@0.7.2`` + * - NuGet (.Net) + - ``pkg:nuget/EnterpriseLibrary.Common@6.0.1304`` + * - NPM (NodeJS) + - ``pkg:npm/%40angular/animation@12.3.1`` + +For more detailed information on converting a given artifact into a PURL, see `PURL Specification `_ and `PURL Types `_ + +.. note:: If a repository is not also provided, Macaron will try to discover it based on the artifact purl. For this to work, ``find_repos`` in the configuration file **must be enabled**\. See `Analyzing more dependencies <#more-deps>`_ for more information about the configuration options of the Repository Finding feature. + ------------------------------------------------- Verifying provenance expectations in CUE language ------------------------------------------------- @@ -191,6 +224,8 @@ With the example above, the generated output reports can be seen here: - `micronaut-core.html <../_static/examples/micronaut-projects/micronaut-core/analyze_with_sbom/micronaut-core.html>`__ - `micronaut-core.json <../_static/examples/micronaut-projects/micronaut-core/analyze_with_sbom/micronaut-core.json>`__ +.. _more-deps: + ''''''''''''''''''''''''''' Analyzing more dependencies '''''''''''''''''''''''''''