Improve nuget/rubygems package registries (#34741)

1. Add some missing (optional) fields for nuget v2, and sort the fields
to make it easier to maintain
2. Add missing "platform" for rubygems: `VERSION-PLATFORM` and
`VERSION_PLATFORM`

Co-authored-by: Giteabot <[email protected]>

Author: wxiaoguang

models/packages/container/search.go

@@ -44,7 +44,7 @@ func (opts *BlobSearchOptions) toConds() builder.Cond {
 		cond = cond.And(builder.Eq{"package_version.lower_version": strings.ToLower(opts.Tag)})
 	}
 	if opts.IsManifest {
-		cond = cond.And(builder.Eq{"package_file.lower_name": ManifestFilename})
+		cond = cond.And(builder.Eq{"package_file.lower_name": container_module.ManifestFilename})
 	}
 	if opts.OnlyLead {
 		cond = cond.And(builder.Eq{"package_file.is_lead": true})
@@ -235,7 +235,7 @@ func SearchImageTags(ctx context.Context, opts *ImageTagsSearchOptions) ([]*pack
 func SearchExpiredUploadedBlobs(ctx context.Context, olderThan time.Duration) ([]*packages.PackageFile, error) {
 	var cond builder.Cond = builder.Eq{
 		"package_version.is_internal":   true,
-		"package_version.lower_version": UploadVersion,
+		"package_version.lower_version": container_module.UploadVersion,
 		"package.type":                  packages.TypeContainer,
 	}
 	cond = cond.And(builder.Lt{"package_file.created_unix": time.Now().Add(-olderThan).Unix()})

models/packages/descriptor.go

@@ -103,10 +103,10 @@ func (pd *PackageDescriptor) CalculateBlobSize() int64 {
 
 // GetPackageDescriptor gets the package description for a version
 func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDescriptor, error) {
-	return getPackageDescriptor(ctx, pv, cache.NewEphemeralCache())
+	return GetPackageDescriptorWithCache(ctx, pv, cache.NewEphemeralCache())
 }
 
-func getPackageDescriptor(ctx context.Context, pv *PackageVersion, c *cache.EphemeralCache) (*PackageDescriptor, error) {
+func GetPackageDescriptorWithCache(ctx context.Context, pv *PackageVersion, c *cache.EphemeralCache) (*PackageDescriptor, error) {
 	p, err := cache.GetWithEphemeralCache(ctx, c, "package", pv.PackageID, GetPackageByID)
 	if err != nil {
 		return nil, err
@@ -270,7 +270,7 @@ func GetPackageDescriptors(ctx context.Context, pvs []*PackageVersion) ([]*Packa
 func getPackageDescriptors(ctx context.Context, pvs []*PackageVersion, c *cache.EphemeralCache) ([]*PackageDescriptor, error) {
 	pds := make([]*PackageDescriptor, 0, len(pvs))
 	for _, pv := range pvs {
-		pd, err := getPackageDescriptor(ctx, pv, c)
+		pd, err := GetPackageDescriptorWithCache(ctx, pv, c)
 		if err != nil {
 			return nil, err
 		}

models/packages/container/const.go renamed to modules/packages/container/const.go

@@ -57,14 +57,24 @@ type Package struct {
 
 // Metadata represents the metadata of a Nuget package
 type Metadata struct {
-	Description              string                  `json:"description,omitempty"`
-	ReleaseNotes             string                  `json:"release_notes,omitempty"`
-	Readme                   string                  `json:"readme,omitempty"`
-	Authors                  string                  `json:"authors,omitempty"`
-	ProjectURL               string                  `json:"project_url,omitempty"`
-	RepositoryURL            string                  `json:"repository_url,omitempty"`
-	RequireLicenseAcceptance bool                    `json:"require_license_acceptance"`
-	Dependencies             map[string][]Dependency `json:"dependencies,omitempty"`
+	Authors                  string `json:"authors,omitempty"`
+	Copyright                string `json:"copyright,omitempty"`
+	Description              string `json:"description,omitempty"`
+	DevelopmentDependency    bool   `json:"development_dependency,omitempty"`
+	IconURL                  string `json:"icon_url,omitempty"`
+	Language                 string `json:"language,omitempty"`
+	LicenseURL               string `json:"license_url,omitempty"`
+	MinClientVersion         string `json:"min_client_version,omitempty"`
+	Owners                   string `json:"owners,omitempty"`
+	ProjectURL               string `json:"project_url,omitempty"`
+	Readme                   string `json:"readme,omitempty"`
+	ReleaseNotes             string `json:"release_notes,omitempty"`
+	RepositoryURL            string `json:"repository_url,omitempty"`
+	RequireLicenseAcceptance bool   `json:"require_license_acceptance"`
+	Tags                     string `json:"tags,omitempty"`
+	Title                    string `json:"title,omitempty"`
+
+	Dependencies map[string][]Dependency `json:"dependencies,omitempty"`
 }
 
 // Dependency represents a dependency of a Nuget package
@@ -74,24 +84,30 @@ type Dependency struct {
 }
 
 // https://learn.microsoft.com/en-us/nuget/reference/nuspec
+// https://github.com/NuGet/NuGet.Client/blob/dev/src/NuGet.Core/NuGet.Packaging/compiler/resources/nuspec.xsd
 type nuspecPackage struct {
 	Metadata struct {
-		ID                       string `xml:"id"`
-		Version                  string `xml:"version"`
-		Authors                  string `xml:"authors"`
-		RequireLicenseAcceptance bool   `xml:"requireLicenseAcceptance"`
+		// required fields
+		Authors     string `xml:"authors"`
+		Description string `xml:"description"`
+		ID          string `xml:"id"`
+		Version     string `xml:"version"`
+
+		// optional fields
+		Copyright                string `xml:"copyright"`
+		DevelopmentDependency    bool   `xml:"developmentDependency"`
+		IconURL                  string `xml:"iconUrl"`
+		Language                 string `xml:"language"`
+		LicenseURL               string `xml:"licenseUrl"`
+		MinClientVersion         string `xml:"minClientVersion,attr"`
+		Owners                   string `xml:"owners"`
 		ProjectURL               string `xml:"projectUrl"`
-		Description              string `xml:"description"`
-		ReleaseNotes             string `xml:"releaseNotes"`
 		Readme                   string `xml:"readme"`
-		PackageTypes             struct {
-			PackageType []struct {
-				Name string `xml:"name,attr"`
-			} `xml:"packageType"`
-		} `xml:"packageTypes"`
-		Repository struct {
-			URL string `xml:"url,attr"`
-		} `xml:"repository"`
+		ReleaseNotes             string `xml:"releaseNotes"`
+		RequireLicenseAcceptance bool   `xml:"requireLicenseAcceptance"`
+		Tags                     string `xml:"tags"`
+		Title                    string `xml:"title"`
+
 		Dependencies struct {
 			Dependency []struct {
 				ID      string `xml:"id,attr"`
@@ -107,6 +123,14 @@ type nuspecPackage struct {
 				} `xml:"dependency"`
 			} `xml:"group"`
 		} `xml:"dependencies"`
+		PackageTypes struct {
+			PackageType []struct {
+				Name string `xml:"name,attr"`
+			} `xml:"packageType"`
+		} `xml:"packageTypes"`
+		Repository struct {
+			URL string `xml:"url,attr"`
+		} `xml:"repository"`
 	} `xml:"metadata"`
 }
 
@@ -167,13 +191,23 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) {
 	}
 
 	m := &Metadata{
-		Description:              p.Metadata.Description,
-		ReleaseNotes:             p.Metadata.ReleaseNotes,
 		Authors:                  p.Metadata.Authors,
+		Copyright:                p.Metadata.Copyright,
+		Description:              p.Metadata.Description,
+		DevelopmentDependency:    p.Metadata.DevelopmentDependency,
+		IconURL:                  p.Metadata.IconURL,
+		Language:                 p.Metadata.Language,
+		LicenseURL:               p.Metadata.LicenseURL,
+		MinClientVersion:         p.Metadata.MinClientVersion,
+		Owners:                   p.Metadata.Owners,
 		ProjectURL:               p.Metadata.ProjectURL,
+		ReleaseNotes:             p.Metadata.ReleaseNotes,
 		RepositoryURL:            p.Metadata.Repository.URL,
 		RequireLicenseAcceptance: p.Metadata.RequireLicenseAcceptance,
-		Dependencies:             make(map[string][]Dependency),
+		Tags:                     p.Metadata.Tags,
+		Title:                    p.Metadata.Title,
+
+		Dependencies: make(map[string][]Dependency),
 	}
 
 	if p.Metadata.Readme != "" {
@@ -227,13 +261,13 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) {
 func toNormalizedVersion(v *version.Version) string {
 	var buf bytes.Buffer
 	segments := v.Segments64()
-	fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2])
+	_, _ = fmt.Fprintf(&buf, "%d.%d.%d", segments[0], segments[1], segments[2])
 	if len(segments) > 3 && segments[3] > 0 {
-		fmt.Fprintf(&buf, ".%d", segments[3])
+		_, _ = fmt.Fprintf(&buf, ".%d", segments[3])
 	}
 	pre := v.Prerelease()
 	if pre != "" {
-		fmt.Fprint(&buf, "-", pre)
+		_, _ = fmt.Fprint(&buf, "-", pre)
 	}
 	return buf.String()
 }

modules/packages/nuget/metadata.go

@@ -12,44 +12,62 @@ import (
 )
 
 const (
-	id                = "System.Gitea"
-	semver            = "1.0.1"
-	authors           = "Gitea Authors"
-	projectURL        = "https://gitea.io"
-	description       = "Package Description"
-	releaseNotes      = "Package Release Notes"
-	readme            = "Readme"
-	repositoryURL     = "https://gitea.io/gitea/gitea"
-	targetFramework   = ".NETStandard2.1"
-	dependencyID      = "System.Text.Json"
-	dependencyVersion = "5.0.0"
+	authors                  = "Gitea Authors"
+	copyright                = "Package Copyright"
+	dependencyID             = "System.Text.Json"
+	dependencyVersion        = "5.0.0"
+	developmentDependency    = true
+	description              = "Package Description"
+	iconURL                  = "https://gitea.io/favicon.png"
+	id                       = "System.Gitea"
+	language                 = "Package Language"
+	licenseURL               = "https://gitea.io/license"
+	minClientVersion         = "1.0.0.0"
+	owners                   = "Package Owners"
+	projectURL               = "https://gitea.io"
+	readme                   = "Readme"
+	releaseNotes             = "Package Release Notes"
+	repositoryURL            = "https://gitea.io/gitea/gitea"
+	requireLicenseAcceptance = true
+	tags                     = "tag_1 tag_2 tag_3"
+	targetFramework          = ".NETStandard2.1"
+	title                    = "Package Title"
+	versionStr               = "1.0.1"
 )
 
 const nuspecContent = `<?xml version="1.0" encoding="utf-8"?>
 <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
-  <metadata>
-    <id>` + id + `</id>
-    <version>` + semver + `</version>
-    <authors>` + authors + `</authors>
-    <requireLicenseAcceptance>true</requireLicenseAcceptance>
-    <projectUrl>` + projectURL + `</projectUrl>
-    <description>` + description + `</description>
-    <releaseNotes>` + releaseNotes + `</releaseNotes>
-    <repository url="` + repositoryURL + `" />
-    <readme>README.md</readme>
-    <dependencies>
-      <group targetFramework="` + targetFramework + `">
-        <dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" />
-      </group>
-    </dependencies>
-  </metadata>
+	<metadata minClientVersion="` + minClientVersion + `">
+		<authors>` + authors + `</authors>
+		<copyright>` + copyright + `</copyright>
+		<description>` + description + `</description>
+		<developmentDependency>true</developmentDependency>
+		<iconUrl>` + iconURL + `</iconUrl>
+		<id>` + id + `</id>
+		<language>` + language + `</language>
+		<licenseUrl>` + licenseURL + `</licenseUrl>
+		<owners>` + owners + `</owners>
+		<projectUrl>` + projectURL + `</projectUrl>
+		<readme>README.md</readme>
+		<releaseNotes>` + releaseNotes + `</releaseNotes>
+		<repository url="` + repositoryURL + `" />
+		<requireLicenseAcceptance>true</requireLicenseAcceptance>
+		<tags>` + tags + `</tags>
+		<title>` + title + `</title>
+		<version>` + versionStr + `</version>
+		<dependencies>
+			<group targetFramework="` + targetFramework + `">
+				<dependency id="` + dependencyID + `" version="` + dependencyVersion + `" exclude="Build,Analyzers" />
+			</group>
+		</dependencies>
+	</metadata>
 </package>`
 
 const symbolsNuspecContent = `<?xml version="1.0" encoding="utf-8"?>
 <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
   <metadata>
     <id>` + id + `</id>
-    <version>` + semver + `</version>
+    <version>` + versionStr + `</version>
     <description>` + description + `</description>
     <packageTypes>
       <packageType name="SymbolsPackage" />
@@ -140,14 +158,26 @@ func TestParsePackageMetaData(t *testing.T) {
 		assert.NotNil(t, np)
 		assert.Equal(t, DependencyPackage, np.PackageType)
 
-		assert.Equal(t, id, np.ID)
-		assert.Equal(t, semver, np.Version)
 		assert.Equal(t, authors, np.Metadata.Authors)
-		assert.Equal(t, projectURL, np.Metadata.ProjectURL)
 		assert.Equal(t, description, np.Metadata.Description)
-		assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes)
+		assert.Equal(t, id, np.ID)
+		assert.Equal(t, versionStr, np.Version)
+
+		assert.Equal(t, copyright, np.Metadata.Copyright)
+		assert.Equal(t, developmentDependency, np.Metadata.DevelopmentDependency)
+		assert.Equal(t, iconURL, np.Metadata.IconURL)
+		assert.Equal(t, language, np.Metadata.Language)
+		assert.Equal(t, licenseURL, np.Metadata.LicenseURL)
+		assert.Equal(t, minClientVersion, np.Metadata.MinClientVersion)
+		assert.Equal(t, owners, np.Metadata.Owners)
+		assert.Equal(t, projectURL, np.Metadata.ProjectURL)
 		assert.Equal(t, readme, np.Metadata.Readme)
+		assert.Equal(t, releaseNotes, np.Metadata.ReleaseNotes)
 		assert.Equal(t, repositoryURL, np.Metadata.RepositoryURL)
+		assert.Equal(t, requireLicenseAcceptance, np.Metadata.RequireLicenseAcceptance)
+		assert.Equal(t, tags, np.Metadata.Tags)
+		assert.Equal(t, title, np.Metadata.Title)
+
 		assert.Len(t, np.Metadata.Dependencies, 1)
 		assert.Contains(t, np.Metadata.Dependencies, targetFramework)
 		deps := np.Metadata.Dependencies[targetFramework]
@@ -180,7 +210,7 @@ func TestParsePackageMetaData(t *testing.T) {
 		assert.Equal(t, SymbolsPackage, np.PackageType)
 
 		assert.Equal(t, id, np.ID)
-		assert.Equal(t, semver, np.Version)
+		assert.Equal(t, versionStr, np.Version)
 		assert.Equal(t, description, np.Metadata.Description)
 		assert.Empty(t, np.Metadata.Dependencies)
 	})

modules/packages/nuget/metadata_test.go

@@ -34,7 +34,7 @@ type PortablePdbList []*PortablePdb
 
 func (l PortablePdbList) Close() {
 	for _, pdb := range l {
-		pdb.Content.Close()
+		_ = pdb.Content.Close()
 	}
 }
 
@@ -65,20 +65,20 @@ func ExtractPortablePdb(r io.ReaderAt, size int64) (PortablePdbList, error) {
 
 				buf, err := packages.CreateHashedBufferFromReader(f)
 
-				f.Close()
+				_ = f.Close()
 
 				if err != nil {
 					return err
 				}
 
 				id, err := ParseDebugHeaderID(buf)
 				if err != nil {
-					buf.Close()
+					_ = buf.Close()
 					return fmt.Errorf("Invalid PDB file: %w", err)
 				}
 
 				if _, err := buf.Seek(0, io.SeekStart); err != nil {
-					buf.Close()
+					_ = buf.Close()
 					return err
 				}
 

modules/packages/nuget/symbol_extractor.go

@@ -24,14 +24,14 @@ func TestExtractPortablePdb(t *testing.T) {
 		var buf bytes.Buffer
 		archive := zip.NewWriter(&buf)
 		w, _ := archive.Create(name)
-		w.Write(content)
-		archive.Close()
+		_, _ = w.Write(content)
+		_ = archive.Close()
 		return buf.Bytes()
 	}
 
 	t.Run("MissingPdbFiles", func(t *testing.T) {
 		var buf bytes.Buffer
-		zip.NewWriter(&buf).Close()
+		_ = zip.NewWriter(&buf).Close()
 
 		pdbs, err := ExtractPortablePdb(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
 		assert.ErrorIs(t, err, ErrMissingPdbFiles)

modules/packages/nuget/symbol_extractor_test.go

@@ -130,8 +130,8 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI
 		pv := &packages_model.PackageVersion{
 			PackageID:    p.ID,
 			CreatorID:    pi.Owner.ID,
-			Version:      container_model.UploadVersion,
-			LowerVersion: container_model.UploadVersion,
+			Version:      container_module.UploadVersion,
+			LowerVersion: container_module.UploadVersion,
 			IsInternal:   true,
 			MetadataJSON: "null",
 		}

routers/api/packages/container/blob.go

@@ -151,7 +151,7 @@ func processOciImageManifest(ctx context.Context, mci *manifestCreationInfo, buf
 			return err
 		}
 
-		uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_model.UploadVersion)
+		uploadVersion, err := packages_model.GetInternalVersionByNameAndVersion(ctx, mci.Owner.ID, packages_model.TypeContainer, mci.Image, container_module.UploadVersion)
 		if err != nil && err != packages_model.ErrPackageNotExist {
 			return err
 		}
@@ -492,7 +492,7 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
 	pf, err := createFileFromBlobReference(ctx, pv, nil, &blobReference{
 		Digest:       digest.Digest(manifestDigest),
 		MediaType:    mci.MediaType,
-		Name:         container_model.ManifestFilename,
+		Name:         container_module.ManifestFilename,
 		File:         &packages_model.PackageFileDescriptor{Blob: pb},
 		ExpectedSize: pb.Size,
 		IsLead:       true,
@@ -505,7 +505,7 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack
 		OwnerID:     mci.Owner.ID,
 		PackageType: packages_model.TypeContainer,
 		VersionID:   pv.ID,
-		Query:       container_model.ManifestFilename,
+		Query:       container_module.ManifestFilename,
 	})
 	if err != nil {
 		return nil, false, "", err