Skip to content

Commit ce9ab47

Browse files
authored
Add --latest to mirror only the newest version (#631)
* Add --latest to mirror only the newest version An unversioned PURL mirrors every version the upstream lists. With --latest, or "latest": true in a mirror API job, only the newest is mirrored: the highest by the ecosystem's version ordering, without yanked or retracted versions and, unless there is nothing else, pre-releases. * Pick the latest version with registries.FetchLatestVersion --latest now uses the shared policy from git-pkgs/registries: the version the registry advertises as latest where it has one, otherwise the newest active version by publish time. Deprecated versions are left out too, and newestVersion and the direct vers dependency go away.
1 parent 0b1ca01 commit ce9ab47

7 files changed

Lines changed: 229 additions & 12 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,6 +1156,9 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
11561156
# Mirror all versions of a package
11571157
proxy mirror pkg:npm/lodash
11581158
1159+
# Mirror only the newest version of a package
1160+
proxy mirror --latest pkg:npm/lodash
1161+
11591162
# Mirror from a CycloneDX or SPDX SBOM
11601163
proxy mirror --sbom sbom.cdx.json
11611164

‎cmd/proxy/main.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,7 @@ func runMirror() {
442442
concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
443443
dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
444444
platforms := fs.String("platform", "", "Comma-separated os/arch[/variant] list limiting which platforms of container images are mirrored")
445+
latest := fs.Bool("latest", false, "Mirror only the newest version of PURLs without a version")
445446

446447
fs.Usage = func() {
447448
fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
@@ -450,6 +451,7 @@ func runMirror() {
450451
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash@4.17.21\n")
451452
fmt.Fprintf(os.Stderr, " proxy mirror --sbom sbom.cdx.json\n")
452453
fmt.Fprintf(os.Stderr, " proxy mirror pkg:npm/lodash # all versions\n")
454+
fmt.Fprintf(os.Stderr, " proxy mirror --latest pkg:npm/lodash # newest version\n")
453455
fmt.Fprintf(os.Stderr, " proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20\n\n")
454456
fmt.Fprintf(os.Stderr, "Flags:\n")
455457
fs.PrintDefaults()
@@ -475,9 +477,9 @@ func runMirror() {
475477
fmt.Fprintf(os.Stderr, "error reading SBOM %s: %v\n", *sbomPath, err)
476478
os.Exit(1)
477479
}
478-
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Platforms: platformList}
480+
source = &mirror.SBOMSource{Data: data, Name: *sbomPath, Latest: *latest, Platforms: platformList}
479481
case len(purls) > 0:
480-
source = &mirror.PURLSource{PURLs: purls, Platforms: platformList}
482+
source = &mirror.PURLSource{PURLs: purls, Latest: *latest, Platforms: platformList}
481483
default:
482484
fmt.Fprintf(os.Stderr, "error: provide PURLs or --sbom\n")
483485
fs.Usage()

‎docs/configuration.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1589,6 +1589,14 @@ curl -X POST http://localhost:8080/api/mirror \
15891589
-d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}'
15901590
```
15911591

1592+
`latest` mirrors only the newest version of PURLs without a version, like the command's `--latest` flag:
1593+
1594+
```bash
1595+
curl -X POST http://localhost:8080/api/mirror \
1596+
-H "Content-Type: application/json" \
1597+
-d '{"purls":["pkg:npm/lodash"],"latest":true}'
1598+
```
1599+
15921600
## Cache Delete API
15931601

15941602
Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI:
@@ -1638,6 +1646,7 @@ A CRAN version is mirrored as its source package from `upstream.cran`, for examp
16381646
| `--concurrency` | `4` | Number of parallel downloads |
16391647
| `--dry-run` | `false` | Show what would be mirrored without downloading |
16401648
| `--platform` | | Comma-separated `os/arch[/variant]` list limiting which platforms of container images are mirrored |
1649+
| `--latest` | `false` | Mirror only the newest version of PURLs without a version |
16411650
| `--config` | | Path to configuration file |
16421651
| `--storage-url` | | Storage URL |
16431652
| `--database-driver` | | Database driver |
@@ -1654,6 +1663,12 @@ proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
16541663

16551664
A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran, brew and swift. For cargo and conda the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version. Julia versions are always listed from the General registry.
16561665

1666+
With `--latest`, or `"latest": true` in a mirror API job, only the latest version is mirrored, as git-pkgs/registries picks it: the version the registry advertises as latest where it has one, such as npm's `dist-tags.latest`, and otherwise the newest by publish time, leaving out yanked, retracted and deprecated versions. PURLs with a version, including those in an SBOM, are mirrored as given. Run it from cron to keep the newest releases of a list of packages cached:
1667+
1668+
```bash
1669+
proxy mirror --latest pkg:npm/lodash pkg:pypi/requests pkg:cargo/serde
1670+
```
1671+
16571672
### Container images
16581673

16591674
`pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.

‎internal/mirror/job.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ type JobRequest struct {
4242
// Platforms limit which platforms of container images are mirrored,
4343
// as os/architecture[/variant].
4444
Platforms []string `json:"platforms,omitempty"`
45+
// Latest mirrors only the newest version of PURLs without a version.
46+
Latest bool `json:"latest,omitempty"`
4547
}
4648

4749
// JobStore manages in-memory mirror jobs.
@@ -209,9 +211,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint
209211
case len(req.PURLs) > 0 && len(req.SBOM) > 0:
210212
return nil, fmt.Errorf("request must include only one of purls or sbom")
211213
case len(req.PURLs) > 0:
212-
return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
214+
return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil
213215
case len(req.SBOM) > 0:
214-
return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
216+
return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Latest: req.Latest, Platforms: req.Platforms}, nil
215217
case req.Registry != "":
216218
return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead")
217219
default:

‎internal/mirror/latest_test.go‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package mirror
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"net/http"
7+
"net/http/httptest"
8+
"reflect"
9+
"testing"
10+
)
11+
12+
func TestPURLSourceLatestMirrorsNewestVersion(t *testing.T) {
13+
versions := &fakeVersions{versions: []string{"1.0.0", "1.10.0", "1.2.0"}}
14+
source := &PURLSource{PURLs: []string{"pkg:npm/left-pad", "pkg:npm/is-odd@3.0.0"}, Versions: versions, Latest: true}
15+
16+
items := enumerate(t, source)
17+
18+
want := []PackageVersion{
19+
{Ecosystem: "npm", Name: "left-pad", Version: "1.10.0"},
20+
{Ecosystem: "npm", Name: "is-odd", Version: "3.0.0"},
21+
}
22+
if !reflect.DeepEqual(items, want) {
23+
t.Errorf("items = %v, want %v", items, want)
24+
}
25+
}
26+
27+
func TestPURLSourceLatestWithoutVersions(t *testing.T) {
28+
source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{}, Latest: true}
29+
if items := enumerate(t, source); len(items) != 0 {
30+
t.Errorf("items = %v, want none", items)
31+
}
32+
}
33+
34+
func TestSBOMSourceLatest(t *testing.T) {
35+
bom := map[string]any{
36+
"bomFormat": "CycloneDX",
37+
"specVersion": "1.4",
38+
"components": []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}},
39+
}
40+
source := &SBOMSource{Data: marshalJSON(t, bom), Versions: &fakeVersions{versions: []string{"1.0.0", "1.3.0"}}, Latest: true}
41+
42+
items := enumerate(t, source)
43+
44+
if len(items) != 1 || items[0].Version != "1.3.0" {
45+
t.Errorf("items = %v, want only left-pad@1.3.0", items)
46+
}
47+
}
48+
49+
// TestRegistryVersionsLatestSkipsYanked checks that without an advertised
50+
// latest version the newest version by publish time that isn't yanked wins.
51+
func TestRegistryVersionsLatestSkipsYanked(t *testing.T) {
52+
registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
53+
if r.URL.Path != "/api/v1/crates/demo" {
54+
http.NotFound(w, r)
55+
return
56+
}
57+
_ = json.NewEncoder(w).Encode(map[string]any{
58+
"crate": map[string]any{"name": "demo"},
59+
"versions": []map[string]any{
60+
{"num": "1.3.0", "yanked": true, "created_at": "2026-03-01T00:00:00Z"},
61+
{"num": "1.2.0", "created_at": "2026-02-01T00:00:00Z"},
62+
{"num": "1.10.0-alpha.1", "created_at": "2026-01-01T00:00:00Z"},
63+
},
64+
})
65+
}))
66+
defer registry.Close()
67+
versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }}
68+
69+
got, err := versions.LatestVersion(context.Background(), "cargo", "demo")
70+
if err != nil {
71+
t.Fatalf("LatestVersion() error = %v", err)
72+
}
73+
if got != "1.2.0" {
74+
t.Errorf("LatestVersion() = %q, want 1.2.0", got)
75+
}
76+
}
77+
78+
// TestRegistryVersionsLatestPrefersAdvertised checks that the version the
79+
// registry advertises as latest wins over a higher one, here a 4.0.0
80+
// published by mistake and deprecated.
81+
func TestRegistryVersionsLatestPrefersAdvertised(t *testing.T) {
82+
registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
83+
if r.URL.Path != "/left-pad" {
84+
http.NotFound(w, r)
85+
return
86+
}
87+
_ = json.NewEncoder(w).Encode(map[string]any{
88+
"name": "left-pad",
89+
"dist-tags": map[string]any{"latest": "3.4.0"},
90+
"versions": map[string]any{
91+
"3.4.0": map[string]any{"name": "left-pad", "version": "3.4.0"},
92+
"4.0.0": map[string]any{"name": "left-pad", "version": "4.0.0", "deprecated": "published by mistake"},
93+
},
94+
"time": map[string]any{"3.4.0": "2026-01-01T00:00:00Z", "4.0.0": "2026-02-01T00:00:00Z"},
95+
})
96+
}))
97+
defer registry.Close()
98+
versions := RegistryVersions{URL: func(string, string) (string, error) { return registry.URL, nil }}
99+
100+
got, err := versions.LatestVersion(context.Background(), "npm", "left-pad")
101+
if err != nil {
102+
t.Fatalf("LatestVersion() error = %v", err)
103+
}
104+
if got != "3.4.0" {
105+
t.Errorf("LatestVersion() = %q, want the advertised 3.4.0", got)
106+
}
107+
}
108+
109+
func TestSourceFromRequestPassesLatest(t *testing.T) {
110+
js := NewJobStore(context.Background(), setupTestMirror(t, 1))
111+
112+
source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}, Latest: true})
113+
if err != nil {
114+
t.Fatalf("sourceFromRequest() error = %v", err)
115+
}
116+
if !source.(*PURLSource).Latest {
117+
t.Error("PURL source mirrors every version, want only the newest")
118+
}
119+
120+
source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`), Latest: true})
121+
if err != nil {
122+
t.Fatalf("sourceFromRequest() error = %v", err)
123+
}
124+
if !source.(*SBOMSource).Latest {
125+
t.Error("SBOM source mirrors every version, want only the newest")
126+
}
127+
}

‎internal/mirror/source.go‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"strings"
77

88
"github.com/git-pkgs/purl"
9+
"github.com/git-pkgs/registries"
910
"github.com/git-pkgs/sbom"
1011
)
1112

@@ -46,12 +47,14 @@ type Source interface {
4647

4748
// PURLSource yields packages from PURL strings.
4849
// Versioned PURLs produce a single item. Unversioned PURLs produce one item
49-
// per version Versions lists.
50+
// per version Versions lists, or only the newest with Latest.
5051
type PURLSource struct {
5152
PURLs []string
5253
// Versions lists the versions of unversioned PURLs. Nil lists them from
5354
// the public registries.
5455
Versions VersionLister
56+
// Latest mirrors only the newest version of unversioned PURLs.
57+
Latest bool
5558
// Platforms limit which platforms of container images are mirrored.
5659
Platforms []string
5760
}
@@ -86,8 +89,8 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
8689
continue
8790
}
8891

89-
// Unversioned: enumerate all versions
90-
listed, err := versions.ListVersions(ctx, ecosystem, name)
92+
// Unversioned: enumerate all versions, or the newest
93+
listed, err := s.listVersions(ctx, versions, ecosystem, name)
9194
if err != nil {
9295
return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err)
9396
}
@@ -104,6 +107,36 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
104107
return nil
105108
}
106109

110+
func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, ecosystem, name string) ([]string, error) {
111+
if !s.Latest {
112+
return versions.ListVersions(ctx, ecosystem, name)
113+
}
114+
var latest string
115+
if finder, ok := versions.(LatestFinder); ok {
116+
v, err := finder.LatestVersion(ctx, ecosystem, name)
117+
if err != nil {
118+
return nil, err
119+
}
120+
latest = v
121+
} else {
122+
listed, err := versions.ListVersions(ctx, ecosystem, name)
123+
if err != nil {
124+
return nil, err
125+
}
126+
all := make([]registries.Version, len(listed))
127+
for i, v := range listed {
128+
all[i] = registries.Version{Number: v}
129+
}
130+
if v := registries.SelectLatestVersion(all, purl.EcosystemToPURLType(ecosystem), ""); v != nil {
131+
latest = v.Number
132+
}
133+
}
134+
if latest == "" {
135+
return nil, nil
136+
}
137+
return []string{latest}, nil
138+
}
139+
107140
// mirrorTarget returns the ecosystem, name and file qualifiers a PURL is
108141
// mirrored under.
109142
func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) {
@@ -128,6 +161,8 @@ type SBOMSource struct {
128161
// Versions lists the versions of unversioned PURLs. Nil lists them from
129162
// the public registries.
130163
Versions VersionLister
164+
// Latest mirrors only the newest version of unversioned PURLs.
165+
Latest bool
131166
// Platforms limit which platforms of container images are mirrored.
132167
Platforms []string
133168
}
@@ -141,7 +176,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
141176
return fmt.Errorf("parsing SBOM: %w", err)
142177
}
143178

144-
inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms}
179+
inner := &PURLSource{PURLs: purls, Versions: s.Versions, Latest: s.Latest, Platforms: s.Platforms}
145180
return inner.Enumerate(ctx, fn)
146181
}
147182

‎internal/mirror/versions.go‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,29 @@ type VersionFinder interface {
3636
FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error)
3737
}
3838

39+
// LatestFinder picks the latest version of a package, for unversioned PURLs
40+
// when only the latest is mirrored. Listers without it fall back to
41+
// registries.SelectLatestVersion over ListVersions.
42+
type LatestFinder interface {
43+
LatestVersion(ctx context.Context, ecosystem, name string) (string, error)
44+
}
45+
46+
// LatestVersion implements LatestFinder with registries.FetchLatestVersion:
47+
// the version the registry advertises as latest where it has one, such as
48+
// npm's dist-tags.latest, and otherwise the newest version by publish time
49+
// that isn't yanked, retracted or deprecated.
50+
func (r RegistryVersions) LatestVersion(ctx context.Context, ecosystem, name string) (string, error) {
51+
reg, name, err := r.registryFor(ctx, ecosystem, name)
52+
if err != nil {
53+
return "", err
54+
}
55+
v, err := registries.FetchLatestVersion(ctx, reg, name)
56+
if err != nil || v == nil {
57+
return "", err
58+
}
59+
return v.Number, nil
60+
}
61+
3962
// ListVersions implements VersionLister.
4063
func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) {
4164
versions, err := r.fetchVersions(ctx, ecosystem, name)
@@ -64,10 +87,20 @@ func (r RegistryVersions) FindVersion(ctx context.Context, ecosystem, name, vers
6487
}
6588

6689
func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name string) ([]registries.Version, error) {
67-
reg, err := r.registry(ecosystem, name)
90+
reg, name, err := r.registryFor(ctx, ecosystem, name)
6891
if err != nil {
6992
return nil, err
7093
}
94+
return reg.FetchVersions(ctx, name)
95+
}
96+
97+
// registryFor returns the client for the registry name is listed in, and
98+
// the name the registry knows the package by.
99+
func (r RegistryVersions) registryFor(ctx context.Context, ecosystem, name string) (registries.Registry, string, error) { //nolint:ireturn // registries.New returns the interface
100+
reg, err := r.registry(ecosystem, name)
101+
if err != nil {
102+
return nil, "", err
103+
}
71104
switch purl.EcosystemToPURLType(ecosystem) {
72105
case "helm":
73106
// The namespace names the repository, which URL resolves to the
@@ -76,13 +109,13 @@ func (r RegistryVersions) fetchVersions(ctx context.Context, ecosystem, name str
76109
case "swift":
77110
var baseURL string
78111
if baseURL, err = r.baseURL("swift", name); err != nil {
79-
return nil, err
112+
return nil, "", err
80113
}
81114
if name, err = swiftIdentity(ctx, r.client(), baseURL, name); err != nil {
82-
return nil, err
115+
return nil, "", err
83116
}
84117
}
85-
return reg.FetchVersions(ctx, name)
118+
return reg, name, nil
86119
}
87120

88121
// registry returns the client for the registry name is listed in.

0 commit comments

Comments
 (0)