diff --git a/README.md b/README.md index 0d111ce1..a324c538 100644 --- a/README.md +++ b/README.md @@ -1156,6 +1156,9 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 # Mirror all versions of a package proxy mirror pkg:npm/lodash +# Mirror only the newest version of a package +proxy mirror --latest pkg:npm/lodash + # Mirror from a CycloneDX or SPDX SBOM proxy mirror --sbom sbom.cdx.json diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 21b63881..824f2e2c 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -442,6 +442,7 @@ func runMirror() { concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading") platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored") + latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version") fs.Usage = func() { fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n") @@ -450,6 +451,7 @@ func runMirror() { fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n") fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n") fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n") + fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n") fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n") fmt.Fprintf(os.Stderr, "Flags:\n") fs.PrintDefaults() @@ -475,9 +477,9 @@ func runMirror() { fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err) os.Exit(1) } - source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList} + source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList} case len(purls) > 0: - source = &mirror.PURLSource{PURLs: purls, Platforms: platformList} + source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList} default: fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n") fs.Usage() diff --git a/docs/configuration.md b/docs/configuration.md index 0b9c7799..e25b040e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1589,6 +1589,14 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' ``` +`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag: + +```bash +curl -X POST http://localhost:8080/api/mirror \ + -H "Content-Type: application/json" \ + -d '{"purls":["pkg:npm/lodash"],"latest":true}' +``` + ## Cache Delete API Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: @@ -1638,6 +1646,7 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp | `--concurrency` | `4` | Number of parallel downloads | | `--dry-run` | `false` | Show what would be mirrored without downloading | | `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored | +| `--latest` | `false` | Mirror only the newest version of PURLs without a version | | `--config` | | Path to configuration file | | `--storage-url` | | Storage URL | | `--database-driver` | | Database driver | @@ -1654,6 +1663,12 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0 A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry. +With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached: + +```bash +proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde +``` + ### Container images `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable. diff --git a/internal/mirror/job.go b/internal/mirror/job.go index acf0f251..841c7c9b 100644 --- a/internal/mirror/job.go +++ b/internal/mirror/job.go @@ -42,6 +42,8 @@ type JobRequest struct { // Platforms limit which platforms of container images are mirrored, // as os/architecture[/variant]. Platforms []string `json:"platforms,omitempty"` + // Latest mirrors only the newest version of PURLs without a version. + Latest bool `json:"latest,omitempty"` } // JobStore manages in-memory mirror jobs. @@ -209,9 +211,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint case len(req.PURLs) > 0 && len(req.SBOM) > 0: return nil, fmt.Errorf("request must include only one of purls or sbom") case len(req.PURLs) > 0: - return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case len(req.SBOM) > 0: - return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil + return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil case req.Registry != "": return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead") default: diff --git a/internal/mirror/latest_test.go b/internal/mirror/latest_test.go new file mode 100644 index 00000000..8135116b --- /dev/null +++ b/internal/mirror/latest_test.go @@ -0,0 +1,127 @@ +package mirror + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) { + versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}} + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true} + + items := enumerate(t, source) + + want := []PackageVersion{ + {Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"}, + {Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"}, + } + if !reflect.DeepEqual(items, want) { + t.Errorf("items = %v, want %v", items, want) + } +} + +func TestPURLSourceLatestWithoutVersions(t *testing.T) { + source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true} + if items := enumerate(t, source); len(items) != 0 { + t.Errorf("items = %v, want none", items) + } +} + +func TestSBOMSourceLatest(t *testing.T) { + bom := map[string]any{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}}, + } + source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true} + + items := enumerate(t, source) + + if len(items) != 1 || items[0].Version != "1.3.0" { + t.Errorf("items = %v, want only left-pad@1.3.0", items) + } +} + +// TestRegistryVersionsLatestSkipsYanked checks that without an advertised +// latest version the newest version by publish time that isn't yanked wins. +func TestRegistryVersionsLatestSkipsYanked(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/crates/demo" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "crate": map[string]any{"name": "demo"}, + "versions": []map[string]any{ + {"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"}, + {"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"}, + {"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"}, + }, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "cargo", "demo") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "1.2.0" { + t.Errorf("LatestVersion() = %q, want 1.2.0", got) + } +} + +// TestRegistryVersionsLatestPrefersAdvertised checks that the version the +// registry advertises as latest wins over a higher one, here a 4.0.0 +// published by mistake and deprecated. +func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) { + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/left-pad" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "name": "left-pad", + "dist-tags": map[string]any{"latest": "3.4.0"}, + "versions": map[string]any{ + "3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"}, + "4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"}, + }, + "time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"}, + }) + })) + defer registry.Close() + versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }} + + got, err := versions.LatestVersion(context.Background(), "npm", "left-pad") + if err != nil { + t.Fatalf("LatestVersion() error = %v", err) + } + if got != "3.4.0" { + t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got) + } +} + +func TestSourceFromRequestPassesLatest(t *testing.T) { + js := NewJobStore(context.Background(), setupTestMirror(t, 1)) + + source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*PURLSource).Latest { + t.Error("PURL source mirrors every version, want only the newest") + } + + source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true}) + if err != nil { + t.Fatalf("sourceFromRequest() error = %v", err) + } + if !source.(*SBOMSource).Latest { + t.Error("SBOM source mirrors every version, want only the newest") + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1c8fa7b1..ecc3a0c5 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" ) @@ -46,12 +47,14 @@ type Source interface { // PURLSource yields packages from PURL strings. // Versioned PURLs produce a single item. Unversioned PURLs produce one item -// per version Versions lists. +// per version Versions lists, or only the newest with Latest. type PURLSource struct { PURLs []string // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -86,8 +89,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro continue } - // Unversioned: enumerate all versions - listed, err := versions.ListVersions(ctx, ecosystem, name) + // Unversioned: enumerate all versions, or the newest + listed, err := s.listVersions(ctx, versions, ecosystem, name) if err != nil { return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } @@ -104,6 +107,36 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return nil } +func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) { + if !s.Latest { + return versions.ListVersions(ctx, ecosystem, name) + } + var latest string + if finder, ok := versions.(LatestFinder); ok { + v, err := finder.LatestVersion(ctx, ecosystem, name) + if err != nil { + return nil, err + } + latest = v + } else { + listed, err := versions.ListVersions(ctx, ecosystem, name) + if err != nil { + return nil, err + } + all := make([]registries.Version, len(listed)) + for i, v := range listed { + all[i] = registries.Version{Number: v} + } + if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil { + latest = v.Number + } + } + if latest == "" { + return nil, nil + } + return []string{latest}, nil +} + // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is // mirrored under. func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { @@ -128,6 +161,8 @@ type SBOMSource struct { // Versions lists the versions of unversioned PURLs. Nil lists them from // the public registries. Versions VersionLister + // Latest mirrors only the newest version of unversioned PURLs. + Latest bool // Platforms limit which platforms of container images are mirrored. Platforms []string } @@ -141,7 +176,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("parsing SBOM: %w", err) } - inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms} + inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms} return inner.Enumerate(ctx, fn) } diff --git a/internal/mirror/versions.go b/internal/mirror/versions.go index 8e10edab..04353b47 100644 --- a/internal/mirror/versions.go +++ b/internal/mirror/versions.go @@ -36,6 +36,29 @@ type VersionFinder interface { FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) } +// LatestFinder picks the latest version of a package, for unversioned PURLs +// when only the latest is mirrored. Listers without it fall back to +// registries.SelectLatestVersion over ListVersions. +type LatestFinder interface { + LatestVersion(ctx context.Context, ecosystem, name string) (string, error) +} + +// LatestVersion implements LatestFinder with registries.FetchLatestVersion: +// the version the registry advertises as latest where it has one, such as +// npm's dist-tags.latest, and otherwise the newest version by publish time +// that isn't yanked, retracted or deprecated. +func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) { + reg, name, err := r.registryFor(ctx, ecosystem, name) + if err != nil { + return "", err + } + v, err := registries.FetchLatestVersion(ctx, reg, name) + if err != nil || v == nil { + return "", err + } + return v.Number, nil +} + // ListVersions implements VersionLister. func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { versions, err := r.fetchVersions(ctx, ecosystem, name) @@ -64,10 +87,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers } func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { - reg, err := r.registry(ecosystem, name) + reg, name, err := r.registryFor(ctx, ecosystem, name) if err != nil { return nil, err } + return reg.FetchVersions(ctx, name) +} + +// registryFor returns the client for the registry name is listed in, and +// the name the registry knows the package by. +func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface + reg, err := r.registry(ecosystem, name) + if err != nil { + return nil, "", err + } switch purl.EcosystemToPURLType(ecosystem) { case "helm": // The namespace names the repository, which URL resolves to the @@ -76,13 +109,13 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str case "swift": var baseURL string if baseURL, err = r.baseURL("swift", name); err != nil { - return nil, err + return nil, "", err } if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil { - return nil, err + return nil, "", err } } - return reg.FetchVersions(ctx, name) + return reg, name, nil } // registry returns the client for the registry name is listed in.