From fb1bec88dd921e4e36eb564ce6f4da3d7fc9af15 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 14 Nov 2025 15:59:48 -0500 Subject: [PATCH] Implement PEP 708 fixes: #998 --- CHANGES/998.feature | 1 + .../app/migrations/0018_project_metadata.py | 65 ++++++++ pulp_python/app/models.py | 69 +++++++- pulp_python/app/pypi/views.py | 60 ++++--- pulp_python/app/serializers.py | 47 ++++++ pulp_python/app/tasks/sync.py | 18 +- pulp_python/app/utils.py | 46 ++++- pulp_python/app/viewsets.py | 61 ++++++- pulp_python/pytest_plugin.py | 9 +- .../functional/api/test_project_metadata.py | 157 ++++++++++++++++++ .../api/test_pypi_simple_json_api.py | 7 +- pulp_python/tests/functional/api/test_sync.py | 23 +++ 12 files changed, 525 insertions(+), 38 deletions(-) create mode 100644 CHANGES/998.feature create mode 100644 pulp_python/app/migrations/0018_project_metadata.py create mode 100644 pulp_python/tests/functional/api/test_project_metadata.py diff --git a/CHANGES/998.feature b/CHANGES/998.feature new file mode 100644 index 00000000..0fba5f5a --- /dev/null +++ b/CHANGES/998.feature @@ -0,0 +1 @@ +Implemented PEP 708 support, added new ProjectMetadataContent model to track a package's project level metadata at the repository level. diff --git a/pulp_python/app/migrations/0018_project_metadata.py b/pulp_python/app/migrations/0018_project_metadata.py new file mode 100644 index 00000000..865cec5a --- /dev/null +++ b/pulp_python/app/migrations/0018_project_metadata.py @@ -0,0 +1,65 @@ +# Generated by Django 4.2.26 on 2025-11-13 21:52 + +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion +import pulpcore.app.util + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0145_domainize_import_export"), + ("python", "0017_pythonpackagecontent_size"), + ] + + operations = [ + migrations.AddField( + model_name="pythonremote", + name="project_metadata", + field=models.BooleanField(default=False), + ), + migrations.CreateModel( + name="ProjectMetadataContent", + fields=[ + ( + "content_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.content", + ), + ), + ("project_name", models.TextField()), + ( + "tracks", + django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(), default=list, size=None + ), + ), + ( + "alternate_locations", + django.contrib.postgres.fields.ArrayField( + base_field=models.TextField(), default=list, size=None + ), + ), + ("sha256", models.CharField(max_length=64)), + ( + "_pulp_domain", + models.ForeignKey( + default=pulpcore.app.util.get_domain_pk, + on_delete=django.db.models.deletion.PROTECT, + to="core.domain", + ), + ), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "unique_together": {("sha256", "_pulp_domain")}, + }, + bases=("core.content",), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index d554297f..dbb2eb71 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -1,3 +1,5 @@ +import hashlib +import json from logging import getLogger from aiohttp.web import json_response @@ -5,6 +7,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.conf import settings +from django_lifecycle import hook, BEFORE_SAVE from pulpcore.plugin.models import ( AutoAddObjPermsMixin, Content, @@ -235,6 +238,69 @@ class Meta: ] +class ProjectMetadataContent(Content): + """ + A Content Type representing metadata at the project level. + + Currently used to implement PEP 708. + # TODO: Implement PEP 792 + Fields: + project_name (models.TextField): The name of the project (normalized) + tracks (models.ArrayField): Array of external repository urls that extend the project's + available files (PEP 708) + alternate_locations (models.ArrayField): Array of external repository urls that extends the + project's namespace (PEP 708) + + sha256 (models.CharField): Digest of all the fields above + """ + + TYPE = "project_metadata" + repo_key_fields = ("project_name",) + + project_name = models.TextField() + tracks = ArrayField(models.TextField(), default=list) + alternate_locations = ArrayField(models.TextField(), default=list) + + sha256 = models.CharField(max_length=64, null=False) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) + + @classmethod + def from_simple_page(cls, page): + """Creates a ProjectMetadataContent from a pypi_simple.ProjectPage.""" + metadata_fields = ("alternate_locations", "tracks") + project_metadata = {k: getattr(page, k) for k in metadata_fields if getattr(page, k)} + metadata = cls( + project_name=page.project, + **project_metadata, + ) + metadata.calculate_sha256() + return metadata + + def to_metadata(self): + """Converts model to dict of present fields.""" + return { + "tracks": self.tracks, + "alternate_locations": self.alternate_locations, + } + + @hook(BEFORE_SAVE) + def calculate_sha256(self): + """Calculates the sha256 from the other metadata fields.""" + data = { + "project_name": self.project_name, + "tracks": self.tracks, + "alternate_locations": self.alternate_locations, + } + + metadata_json = json.dumps(data, sort_keys=True).encode("utf-8") + hasher = hashlib.sha256(metadata_json) + self.sha256 = hasher.hexdigest() + + class Meta: + default_related_name = "%(app_label)s_%(model_name)s" + unique_together = ("sha256", "_pulp_domain") + + class PythonPublication(Publication, AutoAddObjPermsMixin): """ A Publication for PythonContent. @@ -270,6 +336,7 @@ class PythonRemote(Remote, AutoAddObjPermsMixin): exclude_platforms = ArrayField( models.CharField(max_length=10, blank=True), choices=PLATFORMS, default=list ) + project_metadata = models.BooleanField(default=False) def get_remote_artifact_url(self, relative_path=None, request=None): """Get url for remote_artifact""" @@ -295,7 +362,7 @@ class PythonRepository(Repository, AutoAddObjPermsMixin): """ TYPE = "python" - CONTENT_TYPES = [PythonPackageContent] + CONTENT_TYPES = [PythonPackageContent, ProjectMetadataContent] REMOTE_TYPES = [PythonRemote] PULL_THROUGH_SUPPORTED = True diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 3fc965f3..c2145e42 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -1,7 +1,5 @@ -import json import logging -from aiohttp.client_exceptions import ClientError from rest_framework.viewsets import ViewSet from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer from rest_framework.response import Response @@ -27,13 +25,12 @@ from packaging.utils import canonicalize_name from urllib.parse import urljoin, urlparse, urlunsplit from pathlib import PurePath -from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.viewsets import OperationPostponedResponse from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.util import get_domain, get_url -from pulpcore.plugin.exceptions import TimeoutException from pulp_python.app.models import ( + ProjectMetadataContent, PythonDistribution, PythonPackageContent, PythonPublication, @@ -53,6 +50,7 @@ PYPI_LAST_SERIAL, PYPI_SERIAL_CONSTANT, get_remote_package_filter, + get_remote_simple_page, ) from pulp_python.app import tasks @@ -120,6 +118,11 @@ def get_content(repository_version): """Returns queryset of the content in this repository version.""" return PythonPackageContent.objects.filter(pk__in=repository_version.content) + @staticmethod + def get_projects_metadata(repository_version): + """Returns queryset of the project metadata in this repository version.""" + return ProjectMetadataContent.objects.filter(pk__in=repository_version.content) + def should_redirect(self, repo_version=None): """Checks if there is a publication the content app can serve.""" if self.distribution.publication: @@ -136,6 +139,12 @@ def get_rvc(self): content = self.get_content(repo_ver) return repo_ver, content + def get_rvcm(self): + """Takes the base_path and returns the repository_version, content, and project metadata.""" + repo_ver, content = self.get_rvc() + project_metadata = self.get_projects_metadata(repo_ver) if repo_ver else None + return repo_ver, content, project_metadata + def initial(self, request, *args, **kwargs): """Perform common initialization tasks for PyPI endpoints.""" super().initial(request, *args, **kwargs) @@ -312,42 +321,37 @@ def parse_package(release_package): rfilter = get_remote_package_filter(remote) if not rfilter.filter_project(package): - return {} + return {}, {} - url = remote.get_remote_artifact_url(f"simple/{package}/") - remote.headers = remote.headers or [] - remote.headers.append({"Accept": ACCEPT_JSON_PREFERRED}) - downloader = remote.get_downloader(url=url, max_retries=1) - try: - d = downloader.fetch() - except (ClientError, TimeoutException): + page = get_remote_simple_page(package, remote) + if not page: log.info(f"Failed to fetch {package} simple page from {remote.url}") - return {} + return {}, {} - if d.headers["content-type"] == PYPI_SIMPLE_V1_JSON: - page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=url) - else: - page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=url) - return { + releases = { p.filename: parse_package(p) for p in page.packages if rfilter.filter_release(package, p.version) } + return releases, ProjectMetadataContent.from_simple_page(page).to_metadata() @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page") def retrieve(self, request, path, package): """Retrieves the simple api html/json page for a package.""" media_type = request.accepted_renderer.media_type - repo_ver, content = self.get_rvc() + repo_ver, content, metadatas = self.get_rvcm() # Should I redirect if the normalized name is different? normalized = canonicalize_name(package) releases = {} + project_metadata = {} if self.distribution.remote: - releases = self.pull_through_package_simple(normalized, path, self.distribution.remote) + releases, project_metadata = self.pull_through_package_simple( + normalized, path, self.distribution.remote + ) elif self.should_redirect(repo_version=repo_ver): return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/")) - if content: + if content is not None: packages = content.filter(name__normalize=normalized).values( "filename", "sha256", @@ -366,17 +370,25 @@ def retrieve(self, request, path, package): for p in packages } releases.update(local_releases) - if not releases: + if metadatas is not None: + local_project_metadata = ( + metadatas.filter(project_name=normalized) + .values("tracks", "alternate_locations") + .first() + ) + if local_project_metadata: + project_metadata.update(local_project_metadata) + if not (releases or project_metadata): return HttpResponseNotFound(f"{normalized} does not exist.") media_type = request.accepted_renderer.media_type headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)} if media_type == PYPI_SIMPLE_V1_JSON: - detail_data = write_simple_detail_json(normalized, releases.values()) + detail_data = write_simple_detail_json(normalized, releases.values(), project_metadata) return Response(detail_data, headers=headers) else: - detail_data = write_simple_detail(normalized, releases.values()) + detail_data = write_simple_detail(normalized, releases.values(), project_metadata) kwargs = {"content_type": media_type, "headers": headers} return HttpResponse(detail_data, **kwargs) diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 6a0e974e..336f3c60 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -16,6 +16,7 @@ artifact_to_python_content_data, get_project_metadata_from_file, parse_project_metadata, + canonicalize_name, ) @@ -464,6 +465,52 @@ class Meta: model = python_models.PythonPackageContent +class ProjectMetadataContentSerializer(core_serializers.NoArtifactContentSerializer): + """ + A Serializer for ProjectMetadataContent. + """ + + project_name = serializers.CharField( + required=True, + help_text=_("The name of the python project."), + ) + tracks = serializers.ListField( + child=serializers.CharField(allow_blank=False), + required=False, + allow_empty=True, + ) + alternate_locations = serializers.ListField( + child=serializers.CharField(allow_blank=False), + required=False, + allow_empty=True, + ) + sha256 = serializers.CharField( + read_only=True, + help_text=_("The SHA256 digest of the project metadata."), + ) + + def validate_project_name(self, value): + """Ensures name is normalized.""" + return canonicalize_name(value) + + def retrieve(self, validated_data): + """Retrieves the project metadata for a project.""" + md = python_models.ProjectMetadataContent(**validated_data) + md.calculate_sha256() + return python_models.ProjectMetadataContent.objects.filter( + sha256=md.sha256, _pulp_domain=get_domain() + ).first() + + class Meta: + fields = core_serializers.NoArtifactContentSerializer.Meta.fields + ( + "project_name", + "tracks", + "alternate_locations", + "sha256", + ) + model = python_models.ProjectMetadataContent + + class MultipleChoiceArrayField(serializers.MultipleChoiceField): """ A wrapper to make sure this DRF serializer works properly with ArrayFields. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index e3ecb108..1632f10b 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -19,8 +19,14 @@ from pulp_python.app.models import ( PythonPackageContent, PythonRemote, + ProjectMetadataContent, +) +from pulp_python.app.utils import ( + parse_metadata, + PYPI_LAST_SERIAL, + get_remote_simple_page, + canonicalize_name, ) -from pulp_python.app.utils import parse_metadata, PYPI_LAST_SERIAL from pypi_simple import IndexPage from bandersnatch.mirror import Mirror @@ -163,6 +169,7 @@ def __init__(self, serial, master, workers, deferred_download, python_stage, pro self.python_stage = python_stage self.progress_report = progress_report self.deferred_download = deferred_download + self.remote = python_stage.remote async def determine_packages_to_sync(self): """ @@ -237,13 +244,20 @@ async def create_content(self, pkg): artifact=artifact, url=url, relative_path=entry["filename"], - remote=self.python_stage.remote, + remote=self.remote, deferred_download=self.deferred_download, ) dc = DeclarativeContent(content=package, d_artifacts=[da]) await self.python_stage.put(dc) + # Create project metadata content if enabled + if self.remote.project_metadata and pkg.releases: + name = canonicalize_name(pkg.name) + if page := get_remote_simple_page(name, self.remote): + if project_metadata_content := ProjectMetadataContent.from_simple_page(page): + await self.python_stage.put(project_metadata_content) + def finalize_sync(self, *args, **kwargs): """No work to be done currently""" pass diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 0fb6ddbf..5e3a0eb3 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -5,6 +5,7 @@ import tempfile import zipfile import json +from aiohttp.client_exceptions import ClientError from collections import defaultdict from django.conf import settings from django.utils import timezone @@ -12,14 +13,18 @@ from packaging.utils import canonicalize_name from packaging.requirements import Requirement from packaging.version import parse, InvalidVersion +from pypi_simple import ACCEPT_JSON_PREFERRED, ProjectPage from pulpcore.plugin.models import Remote +from pulpcore.plugin.exceptions import TimeoutException PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 -SIMPLE_API_VERSION = "1.1" +SIMPLE_API_VERSION = "1.2" +PYPI_SIMPLE_V1_HTML = "application/vnd.pypi.simple.v1+html" +PYPI_SIMPLE_V1_JSON = "application/vnd.pypi.simple.v1+json" simple_index_template = """ @@ -40,6 +45,16 @@ Links for {{ project_name }} + {% if project_metadata.tracks is defined -%} + {%- for url in project_metadata.tracks %} + + {%- endfor -%} + {% endif -%} + {% if project_metadata.alternate_locations is defined -%} + {%- for url in project_metadata.alternate_locations %} + + {%- endfor -%} + {% endif %}

Links for {{ project_name }}

@@ -435,13 +450,15 @@ def write_simple_index(project_names, streamed=False): return simple.stream(**context) if streamed else simple.render(**context) -def write_simple_detail(project_name, project_packages, streamed=False): +def write_simple_detail(project_name, project_packages, project_metadata=None, streamed=False): """Writes the simple detail page of a package.""" detail = Template(simple_detail_template) + project_metadata = project_metadata or {} context = { "SIMPLE_API_VERSION": SIMPLE_API_VERSION, "project_name": project_name, "project_packages": project_packages, + "project_metadata": project_metadata, } return detail.stream(**context) if streamed else detail.render(**context) @@ -456,7 +473,7 @@ def write_simple_index_json(project_names): } -def write_simple_detail_json(project_name, project_packages): +def write_simple_detail_json(project_name, project_packages, project_metadata): """Writes the simple detail page in JSON format.""" return { "meta": {"api-version": SIMPLE_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT}, @@ -484,8 +501,11 @@ def write_simple_detail_json(project_name, project_packages): ], # (v1.1, PEP 700) "versions": sorted(set(package["version"] for package in project_packages)), + # (v1.2, PEP 708) + "alternate-locations": project_metadata.get("alternate_locations", []), + # tracks is only present when there are actual values, else the field is not included + **({"tracks": project_metadata.get("tracks")} if project_metadata.get("tracks") else {}), # TODO in the future: - # alternate-locations (v1.2, PEP 708) # project-status (v1.4, PEP 792 - pypi and docs differ) } @@ -574,3 +594,21 @@ def get_remote_package_filter(remote): rfilter = PackageIncludeFilter(remote) _remote_filters[remote.pulp_id] = (remote.pulp_last_updated, rfilter) return rfilter + + +def get_remote_simple_page(package, remote, max_retries=1): + """Gets the simple page for a package from a remote.""" + url = remote.get_remote_artifact_url(f"simple/{package}/") + remote.headers = remote.headers or [] + remote.headers.append({"Accept": ACCEPT_JSON_PREFERRED}) + downloader = remote.get_downloader(url=url, max_retries=max_retries) + try: + d = downloader.fetch() + except (ClientError, TimeoutException): + return None + + if d.headers["content-type"] == PYPI_SIMPLE_V1_JSON: + page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=url) + else: + page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=url) + return page diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 8aabdfd1..ec9b17dd 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -13,7 +13,8 @@ AsyncOperationResponseSerializer, RepositorySyncURLSerializer, ) -from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.tasking import dispatch, general_create +from pulpcore.plugin.util import get_prn from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers @@ -85,7 +86,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["modify", "repair_metadata"], + "action": ["modify", "repair_metadata", "update_project"], "principal": "authenticated", "effect": "allow", "condition": [ @@ -172,6 +173,41 @@ def sync(self, request, pk): ) return core_viewsets.OperationPostponedResponse(result, request) + @extend_schema( + summary="Update project metadata", responses={202: AsyncOperationResponseSerializer} + ) + @action( + detail=True, + methods=["post"], + serializer_class=python_serializers.ProjectMetadataContentSerializer, + ) + def update_project(self, request, pk): + """ + Update the metadata for a project in the `Repository`. + """ + repository = self.get_object() + serializer = python_serializers.ProjectMetadataContentSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data + + project_metadata = python_models.ProjectMetadataContent.objects.filter( + project_name=serializer.validated_data["project_name"], + pk__in=repository.latest_version().content, + ).first() + if project_metadata: + data = project_metadata.to_metadata() | data + + app_label = python_models.ProjectMetadataContent._meta.app_label + serializer_name = python_serializers.ProjectMetadataContentSerializer.__name__ + data["repository"] = get_prn(repository) + result = dispatch( + general_create, + exclusive_resources=[repository], + args=(app_label, serializer_name), + kwargs={"data": data}, + ) + return core_viewsets.OperationPostponedResponse(result, request) + class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): """ @@ -401,6 +437,27 @@ def upload(self, request): return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) +class ProjectMetadataContentViewSet(core_viewsets.ReadOnlyContentViewSet): + """ + A ViewSet for ProjectMetadataContent. + """ + + endpoint_name = "project_metadata" + queryset = python_models.ProjectMetadataContent.objects.all() + serializer_class = python_serializers.ProjectMetadataContentSerializer + + DEFAULT_ACCESS_POLICY = { + "statements": [ + { + "action": ["list", "retrieve"], + "principal": "authenticated", + "effect": "allow", + }, + ], + "queryset_scoping": {"function": "scope_queryset"}, + } + + class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin): """ diff --git a/pulp_python/pytest_plugin.py b/pulp_python/pytest_plugin.py index 50a32000..1472d2c3 100644 --- a/pulp_python/pytest_plugin.py +++ b/pulp_python/pytest_plugin.py @@ -131,10 +131,11 @@ def python_repo_with_sync( """A factory to generate a Python Repository synced with the passed in Remote.""" def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): - kwargs = {} - if pulp_domain := body.get("pulp_domain"): - kwargs["pulp_domain"] = pulp_domain - remote = remote or python_remote_factory(**kwargs) + remote = remote or {} + if not hasattr(remote, "pulp_href"): + if pulp_domain := body.get("pulp_domain"): + remote["pulp_domain"] = pulp_domain + remote = python_remote_factory(**remote) repo = repository or python_repo_factory(**body) sync_body = {"mirror": mirror, "remote": remote.pulp_href} monitor_task(python_bindings.RepositoriesPythonApi.sync(repo.pulp_href, sync_body).task) diff --git a/pulp_python/tests/functional/api/test_project_metadata.py b/pulp_python/tests/functional/api/test_project_metadata.py new file mode 100644 index 00000000..e31b68f1 --- /dev/null +++ b/pulp_python/tests/functional/api/test_project_metadata.py @@ -0,0 +1,157 @@ +import pytest + +import requests +from urllib.parse import urljoin +from pypi_simple import ProjectPage + + +@pytest.mark.parallel +def test_cru_project_metadata(python_bindings, python_repo_factory, monitor_task): + """Test creating/reading/updating project metadata.""" + repo = python_repo_factory() + body = { + "project_name": "test-project", + } + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata1 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata1.project_name == body["project_name"] + assert metadata1.tracks == [] + assert metadata1.alternate_locations == [] + assert metadata1.sha256 is not None + + # Update metadata + body["alternate_locations"] = ["https://pypi.org/simple/test-project/"] + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata2 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata2.project_name == body["project_name"] + assert metadata2.tracks == [] + assert metadata2.alternate_locations == ["https://pypi.org/simple/test-project/"] + assert metadata2.sha256 is not None + assert metadata1.sha256 != metadata2.sha256 + + # Test that update is a PATCH operation + del body["alternate_locations"] + body["tracks"] = ["https://pypi.org/simple/test-project/"] + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata3 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata3.project_name == body["project_name"] + assert metadata3.tracks == ["https://pypi.org/simple/test-project/"] + assert metadata3.alternate_locations == metadata2.alternate_locations + assert metadata3.sha256 is not None + assert metadata2.sha256 != metadata3.sha256 + + # Test that update is idempotent + body["alternate_locations"] = ["https://pypi.org/simple/test-project/"] + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata4 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata4.pulp_href == metadata3.pulp_href + + # Test name normalization + body["project_name"] = "Test_Project" + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata5 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata5.project_name == "test-project" + assert metadata5.pulp_href == metadata4.pulp_href + + # Test that there is only one project metadata for the repository + repo = python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + assert repo.latest_version_href[-2] == "3" + repo_version = python_bindings.RepositoriesPythonVersionsApi.read(repo.latest_version_href) + assert repo_version.content_summary.present["python.project_metadata"]["count"] == 1 + + +@pytest.mark.parallel +def test_project_metadata_simple( + python_bindings, python_repo_with_sync, python_distribution_factory, monitor_task +): + """Test project metadata is served by the simple API.""" + repo = python_repo_with_sync() + distro = python_distribution_factory(repository=repo) + + tracks = ["https://pypi.org/simple/shelf-reader/"] + alternate_locations = [ + "https://pypi.org/simple/shelf-reader/", + "https://fixtures.pulpproject.org/python-pypi/simple/shelf-reader/", + ] + body = { + "project_name": "shelf-reader", + "tracks": tracks, + "alternate_locations": alternate_locations, + } + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata.project_name == "shelf-reader" + assert metadata.tracks == tracks + assert metadata.alternate_locations == alternate_locations + + # Test that the project metadata is served by the simple API + url = urljoin(distro.base_url, "simple/shelf-reader/") + response = requests.get(url) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "text/html" + page = ProjectPage.from_response(response, "shelf-reader") + assert page.tracks == tracks + assert page.alternate_locations == alternate_locations + + # Test that the project metadata is served by the simple API in JSON format + response = requests.get(url, headers={"Accept": "application/vnd.pypi.simple.v1+json"}) + assert response.status_code == 200 + assert response.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json" + data = response.json() + assert data["tracks"] == tracks + assert data["alternate-locations"] == alternate_locations + + +@pytest.mark.parallel +def test_project_metadata_pull_through( + python_bindings, + python_repo_factory, + python_remote_factory, + python_distribution_factory, + monitor_task, +): + """Test project metadata is served by the pull-through API.""" + repo = python_repo_factory() + distro = python_distribution_factory(repository=repo) + body = { + "project_name": "shelf-reader", + "tracks": ["https://pypi.org/simple/shelf-reader/"], + "alternate_locations": ["https://pypi.org/simple/shelf-reader/"], + } + result = python_bindings.RepositoriesPythonApi.update_project(repo.pulp_href, body) + task = monitor_task(result.task) + metadata = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + + repo2 = python_repo_factory() + remote = python_remote_factory(url=distro.base_url, includes=[]) + distro2 = python_distribution_factory(repository=repo2, remote=remote.pulp_href) + + response = requests.get(urljoin(distro2.base_url, "simple/shelf-reader/")) + assert response.status_code == 200 + page = ProjectPage.from_response(response, "shelf-reader") + assert page.tracks == ["https://pypi.org/simple/shelf-reader/"] + assert page.alternate_locations == ["https://pypi.org/simple/shelf-reader/"] + + # Test that you can override project metadata from pull-through with local repo metadata + body = { + "project_name": "shelf-reader", + "tracks": [], + "alternate_locations": ["http://test.org/simple/shelf-reader/"], + } + # This only adds the metadata, the repository is still empty of packages + result = python_bindings.RepositoriesPythonApi.update_project(repo2.pulp_href, body) + task = monitor_task(result.task) + metadata2 = python_bindings.ContentProjectMetadataApi.read(task.result["pulp_href"]) + assert metadata.sha256 != metadata2.sha256 + + response = requests.get(urljoin(distro2.base_url, "simple/shelf-reader/")) + assert response.status_code == 200 + page = ProjectPage.from_response(response, "shelf-reader") + assert page.tracks == [] + assert page.alternate_locations == ["http://test.org/simple/shelf-reader/"] diff --git a/pulp_python/tests/functional/api/test_pypi_simple_json_api.py b/pulp_python/tests/functional/api/test_pypi_simple_json_api.py index 6a0bfb9f..56d580de 100644 --- a/pulp_python/tests/functional/api/test_pypi_simple_json_api.py +++ b/pulp_python/tests/functional/api/test_pypi_simple_json_api.py @@ -11,7 +11,7 @@ PYTHON_WHEEL_URL, ) -API_VERSION = "1.1" +API_VERSION = "1.2" PYPI_SERIAL_CONSTANT = 1000000000 PYPI_TEXT_HTML = "text/html" @@ -71,6 +71,11 @@ def test_simple_json_detail_api( assert data["files"] assert data["versions"] == ["0.1"] + # Alternate locations is always present even if no metadata is in repository + # Tracks is not present if no metadata is in repository + assert data["alternate_locations"] == [] + assert "tracks" not in data + # Check data of a wheel file_whl = next( (i for i in data["files"] if i["filename"] == "shelf_reader-0.1-py2-none-any.whl"), None diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index 5069b108..eed4cec9 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -324,3 +324,26 @@ def test_proxy_auth_sync( content = python_bindings.ContentPackagesApi.list(repository_version=repo.latest_version_href) assert content.count == 2 + + +@pytest.mark.parallel +def test_sync_project_metadata( + python_bindings, python_repo_with_sync, python_remote_factory, python_content_summary +): + """Test syncing with project metadata.""" + remote = python_remote_factory(project_metadata=True) + repo = python_repo_with_sync(remote) + assert repo.latest_version_href[-2] == "1" + + summary = python_content_summary(repository_version=repo.latest_version_href) + assert summary.present["python.project_metadata"]["count"] == 1 + assert summary.present["python.python"]["count"] == 2 + + metadata = python_bindings.ContentProjectMetadataApi.list( + repository_version=repo.latest_version_href + ) + assert metadata.count == 1 + assert metadata.results[0].project_name == "shelf-reader" + assert metadata.results[0].tracks == [] + assert metadata.results[0].alternate_locations == [] + assert metadata.results[0].sha256 is not None