From bafc25c999b37c589b54dab4629921572e5f1db4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 28 Oct 2026 19:01:50 +0100 Subject: [PATCH 1/2] Add --latest to mirror only the newest version An unversioned PURL mirrors every version the upstream lists. With --latest, or "latest": true in a mirror API job, only the newest is mirrored: the highest by the ecosystem's version ordering, without yanked or retracted versions and, unless there is nothing else, pre-releases. --- README.md | 3 + cmd/proxy/main.go | 6 +- docs/configuration.md | 15 +++++ go.mod | 2 +- internal/mirror/job.go | 6 +- internal/mirror/latest_test.go | 115 +++++++++++++++++++++++++++++++++ internal/mirror/source.go | 36 +++++++++-- internal/mirror/versions.go | 44 +++++++++++++ 8 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 internal/mirror/latest_test.go diff --git a/README.md b/README.md index 48a575a4..a813080e 100644 --- a/README.md +++ b/README.md @@ -1152,6 +1152,9 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 # Mirror all versions of a package proxy mirror pkg:npm/lodash +# Mirror only the newest version of a package +proxy mirror --latest pkg:npm/lodash + # Mirror from a CycloneDX or SPDX SBOM proxy mirror --sbom sbom.cdx.json diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 525a4f6b..9cf6d4ca 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -442,6 +442,7 @@ func runMirror() { concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored") + latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version") fs.Usage = func() { fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") @@ -450,6 +451,7 @@ func runMirror() { fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") + fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n") fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") fs.PrintDefaults() @@ -475,9 +477,9 @@ func runMirror() { fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err) os.Exit(1) } - source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList} + source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList} case len(purls) > 0: - source = &mirror.PURLSource{PURLs: purls, Platforms: platformList} + source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList} default: fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fs.Usage() diff --git a/docs/configuration.md b/docs/configuration.md index 832860c2..fc2b3b3c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1584,6 +1584,14 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' ``` +`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag: + +```bash +curl -X POST http://localhost:8080/api/mirror \ + -H "Content-Type: application/json" \ + -d '{"purls":["pkg:npm/lodash"],"latest":true}' +``` + ## Cache Delete API Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: @@ -1631,6 +1639,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check | `--concurrency` | `4` | Number of parallel downloads | | `--dry-run` | `false` | Show what would be mirrored without downloading | | `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored | +| `--latest` | `false` | Mirror only the newest version of PURLs without a version | | `--config` | | Path to configuration file | | `--storage-url` | | Storage URL | | `--database-driver` | | Database driver | @@ -1647,6 +1656,12 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. +With `--latest`, or `"latest": true` in a mirror API job, only the newest version is mirrored: the highest by the ecosystem's version ordering, leaving out yanked and retracted versions and, unless there is nothing else, pre-releases. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: + +```bash +proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde +``` + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/go.mod b/go.mod index dd44b27a..59e145bd 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.12.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/mirror/job.go b/internal/mirror/job.go index acf0f251..841c7c9b 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -42,6 +42,8 @@ type JobRequest struct { // Platforms limit which platforms of container images are mirrored, // as os/architecture[/variant]. Platforms []string `json:"platforms,omitempty"` + // Latest mirrors only the newest version of PURLs without a version. + Latest bool `json:"latest,omitempty"` } // JobStore manages in-memory mirror jobs. @@ -209,9 +211,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/latest_test.go b/internal/mirror/latest_test.go new file mode 100644 index 00000000..0c77e0ff --- /dev/null +++ b/internal/mirror/latest_test.go @@ -0,0 +1,115 @@ +package mirror + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestNewestVersion(t *testing.T) { + tests := []struct { + ecosystem string + versions []string + want string + }{ + {"npm", []string{"1.9.0", "1.10.0", "2.0.0-rc.1", "1.2.3"}, "1.10.0"}, + {"npm", []string{"2.0.0-beta.1", "2.0.0-beta.2"}, "2.0.0-beta.2"}, + {"pypi", []string{"1.9", "1.10", "2.0rc1", "2.0.dev3"}, "1.10"}, + {"gem", []string{"7.1.0", "7.2.0.beta1", "7.0.8"}, "7.1.0"}, + {"golang", []string{"v1.2.0", "v1.10.0", "v2.0.0-beta.1"}, "v1.10.0"}, + {"maven", []string{"1.0", "1.10", "1.9", "2.0-alpha-1"}, "1.10"}, + {"npm", nil, ""}, + } + for _, tt := range tests { + if got := newestVersion(tt.ecosystem, tt.versions); got != tt.want { + t.Errorf("newestVersion(%s, %v) = %q, want %q", tt.ecosystem, tt.versions, got, tt.want) + } + } +} + +func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.2.0", "1.1.0", "2.0.0-rc.1"}} + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "left-pad", Version: "1.2.0"}, + {Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} + +func TestPURLSourceLatestWithoutVersions(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true} + if items := enumerate(t, source); len(items) != 0 { + t.Errorf("items = %v, want none", items) + } +} + +func TestSBOMSourceLatest(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want only left-pad@1.3.0", items) + } +} + +func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/crates/demo" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "crate": map[string]any{"name": "demo"}, + "versions": []map[string]any{ + {"num": "1.3.0", "yanked": true}, + {"num": "1.2.0"}, + {"num": "1.10.0-alpha.1"}, + }, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "cargo", "demo") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "1.2.0" { + t.Errorf("LatestVersion() = %q, want 1.2.0", got) + } +} + +func TestSourceFromRequestPassesLatest(t *testing.T) { + js := NewJobStore(context.Background(), setupTestMirror(t, 1)) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*PURLSource).Latest { + t.Error("PURL source mirrors every version, want only the newest") + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*SBOMSource).Latest { + t.Error("SBOM source mirrors every version, want only the newest") + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index b742e002..f007e784 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -46,12 +46,14 @@ type Source interface { // PURLSource yields packages from PURL strings. // Versioned PURLs produce a single item. Unversioned PURLs produce one item -// per version Versions lists. +// per version Versions lists, or only the newest with Latest. type PURLSource struct { PURLs []string // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -94,8 +96,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro continue } - // Unversioned: enumerate all versions - listed, err := versions.ListVersions(ctx, ecosystem, name) + // Unversioned: enumerate all versions, or the newest + listed, err := s.listVersions(ctx, versions, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } @@ -108,6 +110,30 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } +func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) { + if !s.Latest { + return versions.ListVersions(ctx, ecosystem, name) + } + var latest string + if finder, ok := versions.(LatestFinder); ok { + v, err := finder.LatestVersion(ctx, ecosystem, name) + if err != nil { + return nil, err + } + latest = v + } else { + listed, err := versions.ListVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + latest = newestVersion(ecosystem, listed) + } + if latest == "" { + return nil, nil + } + return []string{latest}, nil +} + // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data. type SBOMSource struct { Data []byte @@ -115,6 +141,8 @@ type SBOMSource struct { // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -128,7 +156,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8e10edab..55bd6cb2 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -1,6 +1,7 @@ package mirror import ( + "cmp" "context" "encoding/json" "errors" @@ -14,6 +15,7 @@ import ( "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" _ "github.com/git-pkgs/registries/all" + "github.com/git-pkgs/vers" ) // VersionLister lists the versions of a package, for PURLs without one. @@ -36,6 +38,48 @@ type VersionFinder interface { FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } +// LatestFinder picks the newest version of a package, for unversioned PURLs +// when only the newest is mirrored. Listers without it fall back to the +// newest of ListVersions. +type LatestFinder interface { + LatestVersion(ctx context.Context, ecosystem, name string) (string, error) +} + +// LatestVersion implements LatestFinder. Yanked and retracted versions are +// skipped. +func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) { + versions, err := r.fetchVersions(ctx, ecosystem, name) + if err != nil { + return "", err + } + numbers := make([]string, 0, len(versions)) + for _, v := range versions { + if v.Status == registries.StatusYanked || v.Status == registries.StatusRetracted { + continue + } + numbers = append(numbers, v.Number) + } + return newestVersion(ecosystem, numbers), nil +} + +// newestVersion returns the highest of versions by the ecosystem's ordering, +// leaving out pre-releases unless there is nothing else. It returns "" for +// an empty list. +func newestVersion(ecosystem string, versions []string) string { + scheme := purl.EcosystemToPURLType(ecosystem) + var release, prerelease string + for _, v := range versions { + best := &release + if vers.IsPrereleaseWithScheme(v, scheme) { + best = &prerelease + } + if *best == "" || vers.CompareWithScheme(v, *best, scheme) > 0 { + *best = v + } + } + return cmp.Or(release, prerelease) +} + // ListVersions implements VersionLister. func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) From ea2baef8dd1dfad5e4926d726fd766d14214abaa Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 10:20:36 +0100 Subject: [PATCH 2/2] Pick the latest version with registries.FetchLatestVersion --latest now uses the shared policy from git-pkgs/registries: the version the registry advertises as latest where it has one, otherwise the newest active version by publish time. Deprecated versions are left out too, and newestVersion and the direct vers dependency go away. --- docs/configuration.md | 2 +- go.mod | 2 +- internal/mirror/latest_test.go | 64 ++++++++++++++++++++-------------- internal/mirror/source.go | 9 ++++- internal/mirror/versions.go | 63 ++++++++++++++------------------- 5 files changed, 74 insertions(+), 66 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1565eca5..e25b040e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1663,7 +1663,7 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. -With `--latest`, or `"latest": true` in a mirror API job, only the newest version is mirrored: the highest by the ecosystem's version ordering, leaving out yanked and retracted versions and, unless there is nothing else, pre-releases. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: +With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: ```bash proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde diff --git a/go.mod b/go.mod index 0adcd7e9..4a164c29 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/git-pkgs/registries v0.12.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 - github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.3 @@ -77,6 +76,7 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/mirror/latest_test.go b/internal/mirror/latest_test.go index 0c77e0ff..8135116b 100644 --- a/internal/mirror/latest_test.go +++ b/internal/mirror/latest_test.go @@ -9,35 +9,14 @@ import ( "testing" ) -func TestNewestVersion(t *testing.T) { - tests := []struct { - ecosystem string - versions []string - want string - }{ - {"npm", []string{"1.9.0", "1.10.0", "2.0.0-rc.1", "1.2.3"}, "1.10.0"}, - {"npm", []string{"2.0.0-beta.1", "2.0.0-beta.2"}, "2.0.0-beta.2"}, - {"pypi", []string{"1.9", "1.10", "2.0rc1", "2.0.dev3"}, "1.10"}, - {"gem", []string{"7.1.0", "7.2.0.beta1", "7.0.8"}, "7.1.0"}, - {"golang", []string{"v1.2.0", "v1.10.0", "v2.0.0-beta.1"}, "v1.10.0"}, - {"maven", []string{"1.0", "1.10", "1.9", "2.0-alpha-1"}, "1.10"}, - {"npm", nil, ""}, - } - for _, tt := range tests { - if got := newestVersion(tt.ecosystem, tt.versions); got != tt.want { - t.Errorf("newestVersion(%s, %v) = %q, want %q", tt.ecosystem, tt.versions, got, tt.want) - } - } -} - func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) { - versions := &fakeVersions{versions: []string{"1.0.0", "1.2.0", "1.1.0", "2.0.0-rc.1"}} + versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}} source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true} items := enumerate(t, source) want := []PackageVersion{ - {Ecosystem: "npm", Name: "left-pad", Version: "1.2.0"}, + {Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"}, {Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"}, } if !reflect.DeepEqual(items, want) { @@ -67,6 +46,8 @@ func TestSBOMSourceLatest(t *testing.T) { } } +// TestRegistryVersionsLatestSkipsYanked checks that without an advertised +// latest version the newest version by publish time that isn't yanked wins. func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/crates/demo" { @@ -76,9 +57,9 @@ func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { _ = json.NewEncoder(w).Encode(map[string]any{ "crate": map[string]any{"name": "demo"}, "versions": []map[string]any{ - {"num": "1.3.0", "yanked": true}, - {"num": "1.2.0"}, - {"num": "1.10.0-alpha.1"}, + {"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"}, + {"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"}, + {"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"}, }, }) })) @@ -94,6 +75,37 @@ func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { } } +// TestRegistryVersionsLatestPrefersAdvertised checks that the version the +// registry advertises as latest wins over a higher one, here a 4.0.0 +// published by mistake and deprecated. +func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/left-pad" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "left-pad", + "dist-tags": map[string]any{"latest": "3.4.0"}, + "versions": map[string]any{ + "3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"}, + "4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"}, + }, + "time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"}, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "3.4.0" { + t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got) + } +} + func TestSourceFromRequestPassesLatest(t *testing.T) { js := NewJobStore(context.Background(), setupTestMirror(t, 1)) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 6541779c..ecc3a0c5 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" ) @@ -122,7 +123,13 @@ func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, e if err != nil { return nil, err } - latest = newestVersion(ecosystem, listed) + all := make([]registries.Version, len(listed)) + for i, v := range listed { + all[i] = registries.Version{Number: v} + } + if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil { + latest = v.Number + } } if latest == "" { return nil, nil diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 55bd6cb2..04353b47 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -1,7 +1,6 @@ package mirror import ( - "cmp" "context" "encoding/json" "errors" @@ -15,7 +14,6 @@ import ( "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" _ "github.com/git-pkgs/registries/all" - "github.com/git-pkgs/vers" ) // VersionLister lists the versions of a package, for PURLs without one. @@ -38,46 +36,27 @@ type VersionFinder interface { FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } -// LatestFinder picks the newest version of a package, for unversioned PURLs -// when only the newest is mirrored. Listers without it fall back to the -// newest of ListVersions. +// LatestFinder picks the latest version of a package, for unversioned PURLs +// when only the latest is mirrored. Listers without it fall back to +// registries.SelectLatestVersion over ListVersions. type LatestFinder interface { LatestVersion(ctx context.Context, ecosystem, name string) (string, error) } -// LatestVersion implements LatestFinder. Yanked and retracted versions are -// skipped. +// LatestVersion implements LatestFinder with registries.FetchLatestVersion: +// the version the registry advertises as latest where it has one, such as +// npm's dist-tags.latest, and otherwise the newest version by publish time +// that isn't yanked, retracted or deprecated. func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) { - versions, err := r.fetchVersions(ctx, ecosystem, name) + reg, name, err := r.registryFor(ctx, ecosystem, name) if err != nil { return "", err } - numbers := make([]string, 0, len(versions)) - for _, v := range versions { - if v.Status == registries.StatusYanked || v.Status == registries.StatusRetracted { - continue - } - numbers = append(numbers, v.Number) - } - return newestVersion(ecosystem, numbers), nil -} - -// newestVersion returns the highest of versions by the ecosystem's ordering, -// leaving out pre-releases unless there is nothing else. It returns "" for -// an empty list. -func newestVersion(ecosystem string, versions []string) string { - scheme := purl.EcosystemToPURLType(ecosystem) - var release, prerelease string - for _, v := range versions { - best := &release - if vers.IsPrereleaseWithScheme(v, scheme) { - best = &prerelease - } - if *best == "" || vers.CompareWithScheme(v, *best, scheme) > 0 { - *best = v - } + v, err := registries.FetchLatestVersion(ctx, reg, name) + if err != nil || v == nil { + return "", err } - return cmp.Or(release, prerelease) + return v.Number, nil } // ListVersions implements VersionLister. @@ -108,10 +87,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - reg, err := r.registry(ecosystem, name) + reg, name, err := r.registryFor(ctx, ecosystem, name) if err != nil { return nil, err } + return reg.FetchVersions(ctx, name) +} + +// registryFor returns the client for the registry name is listed in, and +// the name the registry knows the package by. +func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, "", err + } switch purl.EcosystemToPURLType(ecosystem) { case "helm": // The namespace names the repository, which URL resolves to the @@ -120,13 +109,13 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str case "swift": var baseURL string if baseURL, err = r.baseURL("swift", name); err != nil { - return nil, err + return nil, "", err } if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil { - return nil, err + return nil, "", err } } - return reg.FetchVersions(ctx, name) + return reg, name, nil } // registry returns the client for the registry name is listed in.