Add --latest to mirror only the newest version

README.md

@@ -1156,6 +1156,9 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
 # Mirror all versions of a package
 proxy mirror pkg:npm/lodash
 
+# Mirror only the newest version of a package
+proxy mirror --latest pkg:npm/lodash
+
 # Mirror from a CycloneDX or SPDX SBOM
 proxy mirror --sbom sbom.cdx.json
 

cmd/proxy/main.go

@@ -442,6 +442,7 @@ func runMirror() {
 	concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
 	dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
 	platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
+	latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version")
 
 	fs.Usage = func() {
 		fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
@@ -450,6 +451,7 @@ func runMirror() {
 		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash@4.17.21\n")
 		fmt.Fprintf(os.Stderr, "  proxy mirror --sbom sbom.cdx.json\n")
 		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash  # all versions\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror --latest pkg:npm/lodash  # newest version\n")
 		fmt.Fprintf(os.Stderr, "  proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
 		fmt.Fprintf(os.Stderr, "Flags:\n")
 		fs.PrintDefaults()
@@ -475,9 +477,9 @@ func runMirror() {
 			fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
 			os.Exit(1)
 		}
-		source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList}
+		source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList}
 	case len(purls) > 0:
-		source = &mirror.PURLSource{PURLs: purls, Platforms: platformList}
+		source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList}
 	default:
 		fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
 		fs.Usage()

docs/configuration.md

@@ -1589,6 +1589,14 @@ curl -X POST http://localhost:8080/api/mirror \
   -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}'
 ```
 
+`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag:
+
+```bash
+curl -X POST http://localhost:8080/api/mirror \
+  -H "Content-Type: application/json" \
+  -d '{"purls":["pkg:npm/lodash"],"latest":true}'
+```
+
 ## Cache Delete API
 
 Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI:
@@ -1638,6 +1646,7 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp
 | `--concurrency` | `4` | Number of parallel downloads |
 | `--dry-run` | `false` | Show what would be mirrored without downloading |
 | `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored |
+| `--latest` | `false` | Mirror only the newest version of PURLs without a version |
 | `--config` | | Path to configuration file |
 | `--storage-url` | | Storage URL |
 | `--database-driver` | | Database driver |
@@ -1654,6 +1663,12 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
 
 A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry.
 
+With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached:
+
+```bash
+proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde
+```
+
 ### Container images
 
 `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.

internal/mirror/job.go

@@ -42,6 +42,8 @@ type JobRequest struct {
 	// Platforms limit which platforms of container images are mirrored,
 	// as os/architecture[/variant].
 	Platforms []string `json:"platforms,omitempty"`
+	// Latest mirrors only the newest version of PURLs without a version.
+	Latest bool `json:"latest,omitempty"`
 }
 
 // JobStore manages in-memory mirror jobs.
@@ -209,9 +211,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint
 	case len(req.PURLs) > 0 && len(req.SBOM) > 0:
 		return nil, fmt.Errorf("request must include only one of purls or sbom")
 	case len(req.PURLs) > 0:
-		return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
+		return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil
 	case len(req.SBOM) > 0:
-		return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
+		return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil
 	case req.Registry != "":
 		return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead")
 	default:

internal/mirror/latest_test.go

@@ -0,0 +1,127 @@
+package mirror
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) {
+	versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}}
+	source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true}
+
+	items := enumerate(t, source)
+
+	want := []PackageVersion{
+		{Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"},
+		{Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"},
+	}
+	if !reflect.DeepEqual(items, want) {
+		t.Errorf("items = %v, want %v", items, want)
+	}
+}
+
+func TestPURLSourceLatestWithoutVersions(t *testing.T) {
+	source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true}
+	if items := enumerate(t, source); len(items) != 0 {
+		t.Errorf("items = %v, want none", items)
+	}
+}
+
+func TestSBOMSourceLatest(t *testing.T) {
+	bom := map[string]any{
+		"bomFormat":   "CycloneDX",
+		"specVersion": "1.4",
+		"components":  []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}},
+	}
+	source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true}
+
+	items := enumerate(t, source)
+
+	if len(items) != 1 || items[0].Version != "1.3.0" {
+		t.Errorf("items = %v, want only left-pad@1.3.0", items)
+	}
+}
+
+// TestRegistryVersionsLatestSkipsYanked checks that without an advertised
+// latest version the newest version by publish time that isn't yanked wins.
+func TestRegistryVersionsLatestSkipsYanked(t *testing.T) {
+	registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/v1/crates/demo" {
+			http.NotFound(w, r)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"crate": map[string]any{"name": "demo"},
+			"versions": []map[string]any{
+				{"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"},
+				{"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"},
+				{"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"},
+			},
+		})
+	}))
+	defer registry.Close()
+	versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }}
+
+	got, err := versions.LatestVersion(context.Background(), "cargo", "demo")
+	if err != nil {
+		t.Fatalf("LatestVersion() error = %v", err)
+	}
+	if got != "1.2.0" {
+		t.Errorf("LatestVersion() = %q, want 1.2.0", got)
+	}
+}
+
+// TestRegistryVersionsLatestPrefersAdvertised checks that the version the
+// registry advertises as latest wins over a higher one, here a 4.0.0
+// published by mistake and deprecated.
+func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) {
+	registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/left-pad" {
+			http.NotFound(w, r)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"name":      "left-pad",
+			"dist-tags": map[string]any{"latest": "3.4.0"},
+			"versions": map[string]any{
+				"3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"},
+				"4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"},
+			},
+			"time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"},
+		})
+	}))
+	defer registry.Close()
+	versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }}
+
+	got, err := versions.LatestVersion(context.Background(), "npm", "left-pad")
+	if err != nil {
+		t.Fatalf("LatestVersion() error = %v", err)
+	}
+	if got != "3.4.0" {
+		t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got)
+	}
+}
+
+func TestSourceFromRequestPassesLatest(t *testing.T) {
+	js := NewJobStore(context.Background(), setupTestMirror(t, 1))
+
+	source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true})
+	if err != nil {
+		t.Fatalf("sourceFromRequest() error = %v", err)
+	}
+	if !source.(*PURLSource).Latest {
+		t.Error("PURL source mirrors every version, want only the newest")
+	}
+
+	source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true})
+	if err != nil {
+		t.Fatalf("sourceFromRequest() error = %v", err)
+	}
+	if !source.(*SBOMSource).Latest {
+		t.Error("SBOM source mirrors every version, want only the newest")
+	}
+}

internal/mirror/source.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/git-pkgs/purl"
+	"github.com/git-pkgs/registries"
 	"github.com/git-pkgs/sbom"
 )
 
@@ -46,12 +47,14 @@ type Source interface {
 
 // PURLSource yields packages from PURL strings.
 // Versioned PURLs produce a single item. Unversioned PURLs produce one item
-// per version Versions lists.
+// per version Versions lists, or only the newest with Latest.
 type PURLSource struct {
 	PURLs []string
 	// Versions lists the versions of unversioned PURLs. Nil lists them from
 	// the public registries.
 	Versions VersionLister
+	// Latest mirrors only the newest version of unversioned PURLs.
+	Latest bool
 	// Platforms limit which platforms of container images are mirrored.
 	Platforms []string
 }
@@ -86,8 +89,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 			continue
 		}
 
-		// Unversioned: enumerate all versions
-		listed, err := versions.ListVersions(ctx, ecosystem, name)
+		// Unversioned: enumerate all versions, or the newest
+		listed, err := s.listVersions(ctx, versions, ecosystem, name)
 		if err != nil {
 			return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err)
 		}
@@ -104,6 +107,36 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 	return nil
 }
 
+func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) {
+	if !s.Latest {
+		return versions.ListVersions(ctx, ecosystem, name)
+	}
+	var latest string
+	if finder, ok := versions.(LatestFinder); ok {
+		v, err := finder.LatestVersion(ctx, ecosystem, name)
+		if err != nil {
+			return nil, err
+		}
+		latest = v
+	} else {
+		listed, err := versions.ListVersions(ctx, ecosystem, name)
+		if err != nil {
+			return nil, err
+		}
+		all := make([]registries.Version, len(listed))
+		for i, v := range listed {
+			all[i] = registries.Version{Number: v}
+		}
+		if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil {
+			latest = v.Number
+		}
+	}
+	if latest == "" {
+		return nil, nil
+	}
+	return []string{latest}, nil
+}
+
 // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is
 // mirrored under.
 func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) {
@@ -128,6 +161,8 @@ type SBOMSource struct {
 	// Versions lists the versions of unversioned PURLs. Nil lists them from
 	// the public registries.
 	Versions VersionLister
+	// Latest mirrors only the newest version of unversioned PURLs.
+	Latest bool
 	// Platforms limit which platforms of container images are mirrored.
 	Platforms []string
 }
@@ -141,7 +176,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 		return fmt.Errorf("parsing SBOM: %w", err)
 	}
 
-	inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms}
+	inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms}
 	return inner.Enumerate(ctx, fn)
 }
 

internal/mirror/versions.go

@@ -36,6 +36,29 @@ type VersionFinder interface {
 	FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error)
 }
 
+// LatestFinder picks the latest version of a package, for unversioned PURLs
+// when only the latest is mirrored. Listers without it fall back to
+// registries.SelectLatestVersion over ListVersions.
+type LatestFinder interface {
+	LatestVersion(ctx context.Context, ecosystem, name string) (string, error)
+}
+
+// LatestVersion implements LatestFinder with registries.FetchLatestVersion:
+// the version the registry advertises as latest where it has one, such as
+// npm's dist-tags.latest, and otherwise the newest version by publish time
+// that isn't yanked, retracted or deprecated.
+func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) {
+	reg, name, err := r.registryFor(ctx, ecosystem, name)
+	if err != nil {
+		return "", err
+	}
+	v, err := registries.FetchLatestVersion(ctx, reg, name)
+	if err != nil || v == nil {
+		return "", err
+	}
+	return v.Number, nil
+}
+
 // ListVersions implements VersionLister.
 func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) {
 	versions, err := r.fetchVersions(ctx, ecosystem, name)
@@ -64,10 +87,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers
 }
 
 func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) {
-	reg, err := r.registry(ecosystem, name)
+	reg, name, err := r.registryFor(ctx, ecosystem, name)
 	if err != nil {
 		return nil, err
 	}
+	return reg.FetchVersions(ctx, name)
+}
+
+// registryFor returns the client for the registry name is listed in, and
+// the name the registry knows the package by.
+func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface
+	reg, err := r.registry(ecosystem, name)
+	if err != nil {
+		return nil, "", err
+	}
 	switch purl.EcosystemToPURLType(ecosystem) {
 	case "helm":
 		// The namespace names the repository, which URL resolves to the
@@ -76,13 +109,13 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str
 	case "swift":
 		var baseURL string
 		if baseURL, err = r.baseURL("swift", name); err != nil {
-			return nil, err
+			return nil, "", err
 		}
 		if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil {
-			return nil, err
+			return nil, "", err
 		}
 	}
-	return reg.FetchVersions(ctx, name)
+	return reg, name, nil
 }
 
 // registry returns the client for the registry name is listed in.