From e85a0919ecad4ba511dc16fb79bad1ae23ac2bdc Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 16:54:57 +0100 Subject: [PATCH 1/7] Record versions cooldown filters hold back Part of #617. The package page lists a held back version only if the versions table has it, and the metadata filters didn't write one, so a version nobody downloaded wasn't shown, and a package nobody downloaded had no page. recordWithheld is the shared helper the filters call for each version they hold back. It stores the publish time with DB.RecordWithheldVersion, which adds the package row if it is missing and fills published_at only where it is empty, in one transaction and without reading first. Called from the npm, Cargo, Composer, pub, RubyGems, Hex, Go, Maven, NuGet and PyPI filters. Conda builds have their own times and Helm charts are stored per repository, so neither records; nor does a Maven named repository, whose times are kept under its own PURLs. --- docs/configuration.md | 2 +- internal/database/database_test.go | 73 ++++++ internal/database/queries.go | 37 +++ internal/handler/cargo.go | 1 + internal/handler/composer.go | 1 + internal/handler/cooldown_times.go | 10 +- internal/handler/cooldown_withheld.go | 28 +++ internal/handler/cooldown_withheld_test.go | 280 +++++++++++++++++++++ internal/handler/gem.go | 3 +- internal/handler/go_cooldown.go | 3 + internal/handler/hex.go | 3 +- internal/handler/maven_cooldown.go | 5 + internal/handler/npm.go | 3 + internal/handler/nuget_cooldown.go | 18 +- internal/handler/nuget_registration.go | 6 +- internal/handler/pub.go | 1 + internal/handler/pypi.go | 1 + 17 files changed, 460 insertions(+), 15 deletions(-) create mode 100644 internal/handler/cooldown_withheld.go create mode 100644 internal/handler/cooldown_withheld_test.go diff --git a/docs/configuration.md b/docs/configuration.md index f30f242f..be768401 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1280,7 +1280,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. A Maven named repository doesn't record them either, since its publish times are kept apart from the default repository's. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 500ad0d8..43bcb04e 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1245,5 +1245,78 @@ func TestVersionPublishedAtIsUTC(t *testing.T) { t.Fatal(err) } check("pkg:cargo/demo@2.0.0") + + if err := db.RecordWithheldVersion(&Package{PURL: "pkg:cargo/demo", Ecosystem: "cargo", Name: "demo"}, + "pkg:cargo/demo@3.0.0", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@3.0.0") + }) +} + +// A version a cooldown filter holds back is recorded with its publish time, +// and its package gets a row so the package page exists before anything is +// downloaded. Rows already there keep what they have. +func TestRecordWithheldVersion(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + published := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@2.0.0", published); err != nil { + t.Fatalf("RecordWithheldVersion: %v", err) + } + got, err := db.GetPackageByEcosystemName("npm", "leftpad") + if err != nil || got == nil { + t.Fatalf("package row = %v, %v", got, err) + } + if got.PURL != pkgPURL { + t.Errorf("package purl = %q, want %q", got.PURL, pkgPURL) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil { + t.Fatalf("version row = %v, %v", ver, err) + } + if ver.PackagePURL != pkgPURL || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version = %+v, want package %s published %v", ver, pkgPURL, published) + } + }) +} + +func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + stored := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + other := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad", + Description: sql.NullString{String: "pads left", Valid: true}}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.SetVersionPublishedAt(pkgPURL+"@1.0.0", pkgPURL, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: pkgPURL + "@3.0.0", PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + for _, v := range []string{"1.0.0", "3.0.0"} { + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@"+v, other); err != nil { + t.Fatalf("RecordWithheldVersion %s: %v", v, err) + } + } + + got, err := db.GetPackageByPURL(pkgPURL) + if err != nil || got == nil || got.Description.String != "pads left" { + t.Errorf("package = %+v, %v; want the description kept", got, err) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@1.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Time.Equal(stored) { + t.Errorf("1.0.0 = %+v, %v; want the stored time %v kept", ver, err, stored) + } + ver, err = db.GetVersionByPURL(pkgPURL + "@3.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(other) { + t.Errorf("3.0.0 = %+v, %v; want the empty time filled with %v", ver, err, other) + } }) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 6f693178..2b52543f 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -225,6 +225,43 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// RecordWithheldVersion records a version a cooldown filter held back: its +// publish time, and a row for its package so the package page exists before +// anything is downloaded. It only adds what is missing. An existing package +// row is left as it is, and a stored publish time is not overwritten, so the +// filters can call it for every version they hold back without reading +// first. +func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error { + now := time.Now() + tx, err := db.Beginx() + if err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.Exec(db.Rebind(` + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO NOTHING + `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { + return fmt.Errorf("recording withheld version's package: %w", err) + } + if _, err := tx.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + `), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + return nil +} + // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c742145..337c8031 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -180,6 +180,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) + h.proxy.recordWithheld("cargo", crate.Name, crate.Version, publishedAt) continue } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6b82cfff..8c868fb4 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -409,6 +409,7 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) + h.proxy.recordWithheld("composer", packageName, version, publishedAt) return true } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index b765f9a5..fa9120da 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -138,8 +138,8 @@ func (d timedDocuments) rewriteDigest() string { } // withheld reports whether cooldown leaves version out, given the -// package's publish times. -func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool { +// package's publish times, and records a version it leaves out. +func (d timedDocuments) withheld(p *Proxy, ecosystem, name, version string) bool { published, ok := d.published[version] if !ok { return d.withholdUnknown @@ -147,7 +147,11 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) + if p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), published) { + return false + } + p.recordWithheld(ecosystem, name, version, published) + return true } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go new file mode 100644 index 00000000..ea992cfb --- /dev/null +++ b/internal/handler/cooldown_withheld.go @@ -0,0 +1,28 @@ +package handler + +import ( + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// recordWithheld stores the publish time of a version a cooldown filter held +// back, so the package page can list it with when it becomes available, +// also for a package nobody has downloaded yet. name is the package name as +// the ecosystem's downloads store it, so both land on the same package row. +// A version without a name or a known time is skipped, and a failed write is +// only logged: the metadata is served either way. +func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) { + if p.DB == nil || version == "" || publishedAt.IsZero() { + return + } + packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return + } + pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name} + if err := p.DB.RecordWithheldVersion(pkg, versionPURL, publishedAt); err != nil { + p.Logger.Warn("cooldown: failed to record withheld version", + "ecosystem", ecosystem, "name", name, "version", version, "error", err) + } +} diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go new file mode 100644 index 00000000..61f15f2c --- /dev/null +++ b/internal/handler/cooldown_withheld_test.go @@ -0,0 +1,280 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" +) + +// withheldTestProxy returns a proxy with a database and a three day cooldown, +// and the times of a version old enough to pass it and one it holds back. +func withheldTestProxy(t *testing.T) (proxy *Proxy, db *database.DB, old, recent time.Time) { + t.Helper() + proxy, db, _, _ = setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + now := time.Now().UTC().Truncate(time.Second) + return proxy, db, now.Add(-10 * 24 * time.Hour), now.Add(-time.Hour) +} + +// assertWithheldRecorded checks that a version a filter held back is stored +// with its publish time, under a package row the package page finds by +// ecosystem and name. +func assertWithheldRecorded(t *testing.T, db *database.DB, ecosystem, name, version string, published time.Time) { + t.Helper() + pkg, err := db.GetPackageByEcosystemName(ecosystem, name) + if err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil || ver == nil { + t.Fatalf("version %s = %v, %v; want a row", canonicalVersionPURL(ecosystem, name, version), ver, err) + } + if ver.PackagePURL != pkg.PURL { + t.Errorf("version's package = %q, want %q", ver.PackagePURL, pkg.PURL) + } + if !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version %s published %v, want %v", version, ver.PublishedAt, published) + } +} + +// assertNotRecorded checks that a version the filter let through got no row: +// only held back versions are recorded. +func assertNotRecorded(t *testing.T, db *database.DB, ecosystem, name, version string) { + t.Helper() + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil { + t.Fatalf("GetVersionByPURL: %v", err) + } + if ver != nil { + t.Errorf("version %s recorded although cooldown let it through", version) + } +} + +func TestRecordWithheldNeedsADatabase(t *testing.T) { + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Now()) +} + +func TestRecordWithheldSkipsUnknownTimes(t *testing.T) { + proxy, db, _, _ := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Time{}) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a time", pkg, err) + } +} + +// Without a version the version PURL would be the package's own. +func TestRecordWithheldSkipsVersionsWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "", recent) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a version", pkg, err) + } +} + +func TestNPMMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"testpkg","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}}, + "2.0.0":{"name":"testpkg","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "npm", "testpkg", "2.0.0", recent) + assertNotRecorded(t, db, "npm", "testpkg", "1.0.0") +} + +func TestCargoIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &CargoHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + line := func(version string, at time.Time) string { + return `{"name":"serde","vers":"` + version + `","cksum":"abcd","features":{},"yanked":false,"pubtime":"` + at.Format(time.RFC3339) + `"}` + "\n" + } + h.applyCooldownFiltering(httptest.NewRecorder(), []byte(line("1.0.0", old)+line("1.0.1", recent))) + assertWithheldRecorded(t, db, "cargo", "serde", "1.0.1", recent) + assertNotRecorded(t, db, "cargo", "serde", "1.0.0") +} + +func TestComposerMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"packages":{"symfony/console":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "composer", "symfony/console", "6.0.0", recent) + assertNotRecorded(t, db, "composer", "symfony/console", "5.0.0") +} + +func TestPubMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"flutter_bloc","latest":{"version":"2.0.0"},"versions":[ + {"version":"1.0.0","published":"` + old.Format(time.RFC3339) + `","archive_url":"https://pub.dev/1.0.0.tar.gz"}, + {"version":"2.0.0","published":"` + recent.Format(time.RFC3339) + `","archive_url":"https://pub.dev/2.0.0.tar.gz"}]}` + if _, err := h.rewriteMetadata("flutter_bloc", []byte(input)); err != nil { + t.Fatalf("rewriteMetadata: %v", err) + } + assertWithheldRecorded(t, db, "pub", "flutter_bloc", "2.0.0", recent) + assertNotRecorded(t, db, "pub", "flutter_bloc", "1.0.0") +} + +func TestTimedDocumentsRecordWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + docs := timedDocuments{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent, "3.0.0": {}}} + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0", "4.0.0"} { + docs.withheld(proxy, "hex", "testpkg", v) + } + assertWithheldRecorded(t, db, "hex", "testpkg", "2.0.0", recent) + for _, v := range []string{"1.0.0", "3.0.0", "4.0.0"} { + assertNotRecorded(t, db, "hex", "testpkg", v) + } +} + +func TestGemIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + versionsJSON, _ := json.Marshal([]gemVersion{ + {Number: "1.0.0", Platform: "ruby", CreatedAt: old.Format(time.RFC3339)}, + {Number: "2.0.0", Platform: "ruby", CreatedAt: recent.Format(time.RFC3339)}, + }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/info/"): + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n")) + case strings.HasPrefix(r.URL.Path, "/api/v1/versions/"): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(versionsJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + assertWithheldRecorded(t, db, "gem", "testgem", "2.0.0", recent) + assertNotRecorded(t, db, "gem", "testgem", "1.0.0") +} + +// Go stores the publish time of every version it reads, so what the filter +// adds is the package row. +func TestGoListRecordsWithheldVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + +func assertPackageRow(t *testing.T, db *database.DB, ecosystem, name string) { + t.Helper() + if pkg, err := db.GetPackageByEcosystemName(ecosystem, name); err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } +} + +func TestMavenMetadataRecordsWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + for _, v := range []string{"2.0", "2.1"} { + assertWithheldRecorded(t, proxy.DB, "maven", "com.example:lib", v, u.times[v].UTC().Truncate(time.Second)) + } +} + +func TestMavenNamedRepositoryRecordsNoWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + if w := getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if pkg, err := db.GetPackageByEcosystemName("maven", "com.example:lib"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want no row from a named repository", pkg, err) + } +} + +func TestPyPISimpleRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old.Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, pypiSimpleJSON, body), nil + case "/pypi/ruff/json": + body := `{"releases":{"1.0.0":[{"upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + default: + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + return nil, nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// The NuGet listing and the registration each find the held back version. +func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) { + for _, path := range []string{ + "/nuget/v3-flatcontainer/testpkg/index.json", + "/nuget" + nugetRegistrationPath + "testpkg/index.json", + } { + t.Run(path, func(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), path, http.StatusOK) + + assertPackageRow(t, db, "nuget", "testpkg") + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } + if age := time.Since(ver.PublishedAt.Time); age < 47*time.Hour || age > 49*time.Hour { + t.Errorf("2.0.0 published %v ago, want two days", age) + } + assertNotRecorded(t, db, "nuget", "testpkg", "1.0.0") + }) + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index fa7c24ba..25983cdc 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -176,7 +176,6 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return } - packagePURL := canonicalPackagePURL("gem", name) m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -184,7 +183,7 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return nil, nil, err } out, kept := h.filterIndex(body, name, func(version string) bool { - return docs.withheld(h.proxy, "gem", packagePURL, version) + return docs.withheld(h.proxy, "gem", name, version) }) return out, kept, nil }) diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index efcc1e19..f812c924 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -168,6 +168,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s } keep := make([]bool, len(versions)) + times := make([]time.Time, len(versions)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(goInfoFetchLimit) for i, v := range versions { @@ -177,6 +178,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s return fmt.Errorf("reading time of %s: %w", v, err) } keep[i] = h.allowedAt(module, published) + times[i] = published return nil }) } @@ -190,6 +192,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s allowed = append(allowed, v) } else { h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) + h.proxy.recordWithheld("golang", module, v, times[i]) } } return allowed, nil diff --git a/internal/handler/hex.go b/internal/handler/hex.go index d394c4df..17899f10 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -130,7 +130,6 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { return } - packagePURL := canonicalPackagePURL("hex", name) m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -144,7 +143,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { filtered := make(map[string]bool) var kept []string for _, version := range versions { - if docs.withheld(h.proxy, "hex", packagePURL, version) { + if docs.withheld(h.proxy, "hex", name, version) { filtered[version] = true h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version) continue diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 6155e0d7..beccb703 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -330,6 +330,11 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) removed[v] = true + // A named repository's times are its own, under PURLs that name it, + // and the package page shows the default repository's. + if source.name == "" { + h.proxy.recordWithheld("maven", group+":"+artifact, v, published) + } } if len(removed) == 0 { return body, nil diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 3cf3244d..8993db0d 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -736,6 +736,9 @@ func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry OnCooldownFilter: func(version, publishedAt string) { h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, "published", publishedAt) + if at, err := time.Parse(time.RFC3339, publishedAt); err == nil { + h.proxy.recordWithheld("npm", packageName, version, at) + } }, } } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 714f2192..92651e72 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -393,21 +393,27 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str return false, err } _, published := leaf.release() - return h.nugetPublishedAllowed(id, published), nil + return h.nugetPublishedAllowed(id, version, published), nil } // Missing/invalid timestamps retain the existing permissive behavior, but -// fetch and JSON errors must not bypass the policy. -func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { +// fetch and JSON errors must not bypass the policy. A version held back is +// recorded under the lowercase ID and version downloads are stored under. +func (h *NuGetHandler) nugetPublishedAllowed(id, version, published string) bool { when, err := time.Parse(time.RFC3339, published) if err != nil { return true } - return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + name := strings.ToLower(id) + if h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", name), when) { + return true + } + h.proxy.recordWithheld("nuget", name, nugetVersionKey(version), when) + return false } func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) { - if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) { + if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, version, published) { blocked[nugetVersionKey(version)] = true } for i := range document.Items { @@ -436,7 +442,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request return } if applyCooldown && document.Items == nil { - if _, published := document.release(); !h.nugetPublishedAllowed(id, published) { + if version, published := document.release(); !h.nugetPublishedAllowed(id, version, published) { JSONError(w, http.StatusNotFound, "version not found") return } diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go index 01da55de..e32273ff 100644 --- a/internal/handler/nuget_registration.go +++ b/internal/handler/nuget_registration.go @@ -389,7 +389,11 @@ func (rw *nugetRegistrationWriter) node(dec *jsontext.Decoder, depth int) (bool, if n.container { return n.kept > 0, release.version, nil } - return rw.h.nugetPublishedAllowed(rw.id, release.published), release.version, nil + var version string + if len(release.version) >= 2 && release.version[0] == '"' { + version = string(rw.unquote(release.version)) + } + return rw.h.nugetPublishedAllowed(rw.id, version, release.published), release.version, nil } // linksPage reports whether a node at depth is a page of an index whose diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 5a6d426c..1fe8f040 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -288,6 +288,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) + h.proxy.recordWithheld("pub", name, version, publishedAt) return true } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index c94e1515..716e077a 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -326,6 +326,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s for version, publishedAt := range times { if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true + h.proxy.recordWithheld("pypi", name, version, publishedAt) } } From 27d4556ddc52f3218b6cb9f830ab68db967a3e8c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 12 Nov 2026 17:17:55 +0100 Subject: [PATCH 2/7] Skip withheld versions whose times are kept elsewhere Routed npm and Composer packages keep their times apart from the default registry's PURLs, and a Go module with capitals stores its times under the escaped path, so recording them would put times where the cooldown checks read the default registry's, or give one release two rows. A package without a name is skipped too, and a NuGet registration leaf, which carries no version, records the version its path names. --- docs/configuration.md | 2 +- internal/handler/composer.go | 6 +- internal/handler/cooldown_withheld.go | 6 +- internal/handler/cooldown_withheld_test.go | 98 ++++++++++++++++++++++ internal/handler/go_cooldown.go | 13 ++- internal/handler/npm.go | 5 +- internal/handler/nuget_cooldown.go | 25 +++++- 7 files changed, 147 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index be768401..f3601193 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1280,7 +1280,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. A Maven named repository doesn't record them either, since its publish times are kept apart from the default repository's. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. Neither do a Maven named repository and routed npm and Composer packages, whose publish times are kept apart from the default registry's, nor a Go module with capitals in its path, whose times are stored under the escaped path. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 8c868fb4..6e77e315 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -409,7 +409,11 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) - h.proxy.recordWithheld("composer", packageName, version, publishedAt) + // A routed package's times are the route's, and aren't stored under + // the default registry's PURLs (see releaseTimeStored). + if _, routed := h.routes.match(packageName); !routed { + h.proxy.recordWithheld("composer", packageName, version, publishedAt) + } return true } diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go index ea992cfb..422da672 100644 --- a/internal/handler/cooldown_withheld.go +++ b/internal/handler/cooldown_withheld.go @@ -10,10 +10,10 @@ import ( // back, so the package page can list it with when it becomes available, // also for a package nobody has downloaded yet. name is the package name as // the ecosystem's downloads store it, so both land on the same package row. -// A version without a name or a known time is skipped, and a failed write is -// only logged: the metadata is served either way. +// A version without a package name, a version or a known time is skipped, +// and a failed write is only logged: the metadata is served either way. func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) { - if p.DB == nil || version == "" || publishedAt.IsZero() { + if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() { return } packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go index 61f15f2c..062d337c 100644 --- a/internal/handler/cooldown_withheld_test.go +++ b/internal/handler/cooldown_withheld_test.go @@ -278,3 +278,101 @@ func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) { }) } } + +// Without a name the PURLs would name no package, such as pkg:cargo/. +func TestRecordWithheldSkipsPackagesWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("cargo", "", "1.0.0", recent) + if pkg, err := db.GetPackageByEcosystemName("cargo", ""); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a name", pkg, err) + } +} + +// A routed package's times come from its route and are never stored under +// the default registry's PURLs, so the filter doesn't record them either. +func TestNPMMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"@corp/*": "https://npm.corp.example"})} + input := `{"name":"@corp/lib","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"@corp/lib","version":"1.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-1.0.0.tgz"}}, + "2.0.0":{"name":"@corp/lib","version":"2.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("@corp/lib", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "npm", "@corp/lib", "2.0.0") +} + +func TestComposerMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"corp/*": "https://composer.corp.example"})} + input := `{"packages":{"corp/lib":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "composer", "corp/lib", "6.0.0") +} + +// Go keeps publish times under the escaped module path, which differs from +// the module path downloads are stored under when it has capitals. Recording +// such a module would give one release two version rows, so it is skipped. +func TestGoListOfModuleWithCapitalsRecordsNoSecondRow(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/example.com/!mod/@v/list": + _, _ = w.Write([]byte("v1.1.0\n")) + case "/example.com/!mod/@v/v1.1.0.info": + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.1.0", Time: now.Add(-time.Hour)}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() + if w := goGet(h, "/example.com/!mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/!mod", "v1.1.0")); err != nil || ver == nil { + t.Fatalf("time under the escaped path = %v, %v; want it stored", ver, err) + } + assertNotRecorded(t, db, "golang", "example.com/Mod", "v1.1.0") +} + +// A registration leaf carries no version of its own; it is the one its path +// names. +func TestNuGetRegistrationLeafRecordsWithheldVersion(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), "/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound) + + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } +} + +func TestNuGetRegistrationLeafVersion(t *testing.T) { + for path, want := range map[string]string{ + nugetRegistrationPath + "testpkg/2.0.0.json": "2.0.0", + nugetRegistrationPath + "testpkg/index.json": "", + nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": "", + nugetRegistrationPath + "testpkg/2.0.0": "", + "/other/testpkg/2.0.0.json": "", + } { + if got := nugetRegistrationLeafVersion(path); got != want { + t.Errorf("nugetRegistrationLeafVersion(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index f812c924..52d1bd9a 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -192,12 +192,23 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s allowed = append(allowed, v) } else { h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) - h.proxy.recordWithheld("golang", module, v, times[i]) + h.recordWithheld(module, v, times[i]) } } return allowed, nil } +// recordWithheld records a version the list filter held back. Publish times +// are stored under the escaped module path and downloads under the module +// path, so a module whose path has capitals, and the two differ, is skipped +// rather than given a second row for the same release. +func (h *GoHandler) recordWithheld(module, version string, published time.Time) { + if escaped, err := gomodule.EscapePath(module); err != nil || escaped != module { + return + } + h.proxy.recordWithheld("golang", module, version, published) +} + // serveCooldownList serves @v/list without versions in cooldown. func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { allowed, err := h.allowedVersions(r.Context(), escapedModule, module) diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 8993db0d..95308b78 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -721,6 +721,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l // cooldown is evaluated at, and the pushed versions to list. The callbacks // log what the rewrite did. func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { + _, routed := h.routes.match(packageName) return npm.Options{ ProxyURL: h.proxyURL, Cooldown: h.proxy.cooldown(), @@ -736,7 +737,9 @@ func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry OnCooldownFilter: func(version, publishedAt string) { h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, "published", publishedAt) - if at, err := time.Parse(time.RFC3339, publishedAt); err == nil { + // A routed package's times are the route's, and aren't stored + // under the default registry's PURLs (see versionInCooldown). + if at, err := time.Parse(time.RFC3339, publishedAt); err == nil && !routed { h.proxy.recordWithheld("npm", packageName, version, at) } }, diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 92651e72..1e6615a5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -442,7 +442,11 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request return } if applyCooldown && document.Items == nil { - if version, published := document.release(); !h.nugetPublishedAllowed(id, version, published) { + version, published := document.release() + if version == "" { + version = nugetRegistrationLeafVersion(r.URL.Path) + } + if !h.nugetPublishedAllowed(id, version, published) { JSONError(w, http.StatusNotFound, "version not found") return } @@ -475,6 +479,25 @@ func nugetRegistrationID(path string) string { return "" } +// nugetRegistrationLeafVersion returns the version a registration leaf's +// path names ({prefix}{id}/{version}.json), or "" for any other path. A leaf +// as nuget.org serves it carries no version of its own. +func nugetRegistrationLeafVersion(path string) string { + for _, prefix := range nugetRegistrationPrefixes { + rest, ok := strings.CutPrefix(path, prefix) + if !ok { + continue + } + _, leaf, _ := strings.Cut(rest, "/") + version, isJSON := strings.CutSuffix(leaf, ".json") + if !isJSON || version == "" || version == "index" || strings.Contains(version, "/") { + return "" + } + return version + } + return "" +} + // expandNuGetPageTimes gives each page of a registration index that only // links to its leaves the leaves of the page it links to. With bodies not nil // the linked pages themselves are kept in it, by link. Only index pages are From 4b7dbfd1fcd053e78e485a9f0fd464033b6fdfbb Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 14:40:50 +0100 Subject: [PATCH 3/7] Record versions held back on the PyPI JSON API and a refused Go .info The PyPI JSON API filtered releases in cooldown without recording them, so a client reading /pypi/{name}/json left nothing on the package page. The release filter now hands back the upload time it decided on, and the package and version documents record what they hold back. A Go .info in cooldown was refused without storing anything; the version is now recorded before the refusal. --- internal/handler/cooldown_withheld_test.go | 67 ++++++++++++++++++++++ internal/handler/go_cooldown.go | 1 + internal/handler/pypi.go | 26 ++++++--- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go index 062d337c..f7683c80 100644 --- a/internal/handler/cooldown_withheld_test.go +++ b/internal/handler/cooldown_withheld_test.go @@ -186,6 +186,21 @@ func TestGoListRecordsWithheldVersions(t *testing.T) { assertPackageRow(t, proxy.DB, "golang", "example.com/mod") } +// A .info request for a version in cooldown is refused, and the version is +// recorded with the publish time the refusal was based on. +func TestGoRefusedInfoRecordsWithheldVersion(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", w.Code) + } + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.1.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("version = %v, %v; want it stored with its publish time", ver, err) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + func assertPackageRow(t *testing.T, db *database.DB, ecosystem, name string) { t.Helper() if pkg, err := db.GetPackageByEcosystemName(ecosystem, name); err != nil || pkg == nil { @@ -251,6 +266,58 @@ func TestPyPISimpleRecordsWithheldVersions(t *testing.T) { assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") } +// The JSON API filters releases on its own path, so a client that reads it +// rather than the simple index gets the held back version recorded too. +func TestPyPIJSONRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/pypi/ruff/json" { + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + } + body := `{"info":{"name":"ruff","version":"1.0.0"},"releases":{` + + `"1.0.0":[{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Add(-time.Minute).Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), `"2.0.0"`) { + t.Fatalf("2.0.0 was served although it is in cooldown") + } + // A release's time is its newest file's. + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// The version JSON names one release, in urls, and records it when it is +// held back. +func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { + recent := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"urls":[` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/2.0.0/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), "ruff-2.0.0.tar.gz") { + t.Fatalf("the files of 2.0.0 were served although it is in cooldown") + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) +} + // The NuGet listing and the registration each find the held back version. func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) { for _, path := range []string{ diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 52d1bd9a..e788e688 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -234,6 +234,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es var info goInfo if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { if !h.allowedAt(module, info.Time) { + h.recordWithheld(module, info.Version, info.Time) h.refuseCooldown(w, module, info.Version) return } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 716e077a..0a07136a 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -626,9 +626,17 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } for version, files := range releases { - if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) { + denied := h.proxy.versionDenied("pypi", packageName, version) + held, publishedAt := false, time.Time{} + if !denied { + held, publishedAt = h.shouldFilterRelease(packagePURL, files) + } + if denied || held { h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) + if held { + h.proxy.recordWithheld("pypi", packageName, version, publishedAt) + } delete(releases, version) continue } @@ -637,19 +645,20 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } } -// shouldFilterRelease returns true if a release should be excluded due to cooldown. -func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { +// shouldFilterRelease reports whether a release should be excluded due to +// cooldown, and the newest upload time that decided it. +func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) (bool, time.Time) { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() || packagePURL == "" { - return false + return false, time.Time{} } filesArr, ok := files.([]any) if !ok { - return false + return false, time.Time{} } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt), publishedAt } // versionInCooldown reports whether a version is still inside the cooldown @@ -745,7 +754,10 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL info, _ := metadata["info"].(map[string]any) name, _ := info["name"].(string) version, _ := info["version"].(string) - if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) { + if h.proxy.versionDenied("pypi", name, version) { + metadata["urls"] = []any{} + } else if held, publishedAt := h.shouldFilterRelease(packagePURL, urls); held { + h.proxy.recordWithheld("pypi", name, version, publishedAt) metadata["urls"] = []any{} } From 78523aa0a5b75a94aea1323dbe4dbedb2c59cd96 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 14:59:49 +0100 Subject: [PATCH 4/7] Keep the newest upload time of a withheld PyPI release A release's time is its newest file's, and a file can be added after the release was first held back. The PyPI filters now record through RecordWithheldVersionNewest, which replaces an earlier stored time. The other ecosystems keep the first one. --- internal/database/database_test.go | 23 +++++++ internal/database/queries.go | 16 ++++- internal/handler/cooldown_withheld.go | 16 ++++- internal/handler/cooldown_withheld_test.go | 72 ++++++++++++++++++++++ internal/handler/pypi.go | 6 +- 5 files changed, 128 insertions(+), 5 deletions(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 98787aa7..df1a66fc 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1283,6 +1283,29 @@ func TestRecordWithheldVersion(t *testing.T) { }) } +// For an ecosystem whose release time is its newest file's, a later time +// replaces the stored one and an earlier one doesn't. +func TestRecordWithheldVersionNewest(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:pypi/ruff" + first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + later := first.Add(time.Hour) + pkg := &Package{PURL: pkgPURL, Ecosystem: "pypi", Name: "ruff"} + for _, at := range []time.Time{first, later, first} { + if err := db.RecordWithheldVersionNewest(pkg, pkgPURL+"@2.0.0", at); err != nil { + t.Fatalf("RecordWithheldVersionNewest: %v", err) + } + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) { + t.Errorf("version = %+v, %v; want published %v", ver, err, later) + } + if got, err := db.GetPackageByEcosystemName("pypi", "ruff"); err != nil || got == nil { + t.Errorf("package row = %v, %v", got, err) + } + }) +} + func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { const pkgPURL = "pkg:npm/leftpad" diff --git a/internal/database/queries.go b/internal/database/queries.go index 2b52543f..24d75189 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -232,6 +232,20 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt // filters can call it for every version they hold back without reading // first. func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, "versions.published_at IS NULL") +} + +// RecordWithheldVersionNewest is RecordWithheldVersion for an ecosystem whose +// release time is its newest file's, such as PyPI: a later time replaces the +// stored one, since a file can be added to a release after it was held back. +func (db *DB) RecordWithheldVersionNewest(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, + "versions.published_at IS NULL OR versions.published_at < excluded.published_at") +} + +// recordWithheldVersion records a held back version, updating the stored +// publish time where replace, a condition on the existing row, holds. +func (db *DB) recordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time, replace string) error { now := time.Now() tx, err := db.Beginx() if err != nil { @@ -252,7 +266,7 @@ func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedA ON CONFLICT(purl) DO UPDATE SET published_at = excluded.published_at, updated_at = excluded.updated_at - WHERE versions.published_at IS NULL + WHERE `+replace+` `), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil { return fmt.Errorf("recording withheld version: %w", err) } diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go index 422da672..cbf5fad8 100644 --- a/internal/handler/cooldown_withheld.go +++ b/internal/handler/cooldown_withheld.go @@ -13,6 +13,16 @@ import ( // A version without a package name, a version or a known time is skipped, // and a failed write is only logged: the metadata is served either way. func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) { + p.recordWithheldWith(ecosystem, name, version, publishedAt, false) +} + +// recordWithheldNewest is recordWithheld for a release whose time is its +// newest file's: a later time replaces the stored one. +func (p *Proxy) recordWithheldNewest(ecosystem, name, version string, publishedAt time.Time) { + p.recordWithheldWith(ecosystem, name, version, publishedAt, true) +} + +func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) { if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() { return } @@ -21,7 +31,11 @@ func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time return } pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name} - if err := p.DB.RecordWithheldVersion(pkg, versionPURL, publishedAt); err != nil { + record := p.DB.RecordWithheldVersion + if newest { + record = p.DB.RecordWithheldVersionNewest + } + if err := record(pkg, versionPURL, publishedAt); err != nil { p.Logger.Warn("cooldown: failed to record withheld version", "ecosystem", ecosystem, "name", name, "version", version, "error", err) } diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go index f7683c80..8ddcacce 100644 --- a/internal/handler/cooldown_withheld_test.go +++ b/internal/handler/cooldown_withheld_test.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -296,6 +297,77 @@ func TestPyPIJSONRecordsWithheldVersions(t *testing.T) { assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") } +// A release's time is its newest file's, and files can be added to a release +// after it was first held back, so the newer time replaces the stored one. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + files := []string{`{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + first.Format(time.RFC3339) + `"}`} + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"releases":{"2.0.0":[` + strings.Join(files, ",") + `]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + files = append(files, `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"`+added.Format(time.RFC3339)+`"}`) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) +} + +// The simple index and the version JSON also keep the newest upload time. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTimeOnEveryPath(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + for name, path := range map[string]string{"simple": "/simple/ruff/", "version json": "/pypi/ruff/2.0.0/json"} { + t.Run(name, func(t *testing.T) { + times := []time.Time{first} + files := func(key string) string { + var out []string + for i, at := range times { + out = append(out, fmt.Sprintf(`{"filename":"ruff-2.0.0-%d.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0-%d.tar.gz",%q:%q}`, i, i, key, at.Format(time.RFC3339))) + } + return strings.Join(out, ",") + } + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + return pypiHTTPResponse(r, pypiSimpleJSON, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[`+files("upload-time")+`]}`), nil + case "/pypi/ruff/json": + return pypiHTTPResponse(r, "application/json", `{"releases":{"2.0.0":[`+files("upload_time_iso_8601")+`]}}`), nil + default: + return pypiHTTPResponse(r, "application/json", `{"info":{"name":"ruff","version":"2.0.0"},"urls":[`+files("upload_time_iso_8601")+`]}`), nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + times = append(times, added) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) + }) + } +} + // The version JSON names one release, in urls, and records it when it is // held back. func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 0a07136a..ed8546a7 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -326,7 +326,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s for version, publishedAt := range times { if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true - h.proxy.recordWithheld("pypi", name, version, publishedAt) + h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) } } @@ -635,7 +635,7 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) if held { - h.proxy.recordWithheld("pypi", packageName, version, publishedAt) + h.proxy.recordWithheldNewest("pypi", packageName, version, publishedAt) } delete(releases, version) continue @@ -757,7 +757,7 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL if h.proxy.versionDenied("pypi", name, version) { metadata["urls"] = []any{} } else if held, publishedAt := h.shouldFilterRelease(packagePURL, urls); held { - h.proxy.recordWithheld("pypi", name, version, publishedAt) + h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) metadata["urls"] = []any{} } From 5f5fe4741677c1552858adfd90e7ed6939a69eaf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 18:19:59 +0100 Subject: [PATCH 5/7] Keep the PyPI download path from moving an upload time back storeUploadTime wrote the time it read unconditionally, so a download that read an older document than one already recorded could set an earlier time and end cooldown early. It now goes through AdvanceVersionPublishedAt, which only moves the time forward, as the metadata filters do. --- internal/database/database_test.go | 19 +++++++++++++++++++ internal/database/queries.go | 19 +++++++++++++++++++ internal/handler/cooldown_withheld.go | 6 +++--- internal/handler/cooldown_withheld_test.go | 19 +++++++++++++++++++ internal/handler/pypi.go | 11 ++++++----- 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index df1a66fc..398da071 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1306,6 +1306,25 @@ func TestRecordWithheldVersionNewest(t *testing.T) { }) } +// AdvanceVersionPublishedAt only moves a stored time forward, and creates +// the row when there is none. +func TestAdvanceVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:pypi/ruff" + first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + later := first.Add(time.Hour) + for _, at := range []time.Time{first, later, first} { + if err := db.AdvanceVersionPublishedAt(pkgPURL+"@2.0.0", pkgPURL, at); err != nil { + t.Fatalf("AdvanceVersionPublishedAt: %v", err) + } + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) { + t.Errorf("version = %+v, %v; want published %v", ver, err, later) + } + }) +} + func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { const pkgPURL = "pkg:npm/leftpad" diff --git a/internal/database/queries.go b/internal/database/queries.go index 24d75189..10cb43d3 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -225,6 +225,25 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// AdvanceVersionPublishedAt is SetVersionPublishedAt for a time that only +// moves forward, such as a PyPI release's newest upload: an earlier time, +// read from an older document, leaves the stored one. +func (db *DB) AdvanceVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + _, err := db.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL OR versions.published_at < excluded.published_at + `), versionPURL, packagePURL, publishedAt.UTC(), now, now) + if err != nil { + return fmt.Errorf("advancing version publish time: %w", err) + } + return nil +} + // RecordWithheldVersion records a version a cooldown filter held back: its // publish time, and a row for its package so the package page exists before // anything is downloaded. It only adds what is missing. An existing package diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go index cbf5fad8..52c12a6a 100644 --- a/internal/handler/cooldown_withheld.go +++ b/internal/handler/cooldown_withheld.go @@ -16,10 +16,10 @@ func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time p.recordWithheldWith(ecosystem, name, version, publishedAt, false) } -// recordWithheldNewest is recordWithheld for a release whose time is its +// recordWithheldPyPI is recordWithheld for a PyPI release, whose time is its // newest file's: a later time replaces the stored one. -func (p *Proxy) recordWithheldNewest(ecosystem, name, version string, publishedAt time.Time) { - p.recordWithheldWith(ecosystem, name, version, publishedAt, true) +func (p *Proxy) recordWithheldPyPI(name, version string, publishedAt time.Time) { + p.recordWithheldWith("pypi", name, version, publishedAt, true) } func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) { diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go index 8ddcacce..362780e4 100644 --- a/internal/handler/cooldown_withheld_test.go +++ b/internal/handler/cooldown_withheld_test.go @@ -368,6 +368,25 @@ func TestPyPIWithheldReleaseKeepsItsNewestUploadTimeOnEveryPath(t *testing.T) { } } +// A download that read an older document than one already recorded leaves +// the newer upload time, so cooldown doesn't end early. +func TestPyPIDownloadKeepsTheNewerStoredUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + older, newer := now.Add(-2*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", + `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.recordWithheldPyPI("ruff", "2.0.0", newer) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown { + t.Fatalf("versionInCooldown = %v, %v; want true", inCooldown, err) + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", newer) +} + // The version JSON names one release, in urls, and records it when it is // held back. func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index ed8546a7..accd786a 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -326,7 +326,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s for version, publishedAt := range times { if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true - h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) + h.proxy.recordWithheldPyPI(name, version, publishedAt) } } @@ -635,7 +635,7 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) if held { - h.proxy.recordWithheldNewest("pypi", packageName, version, publishedAt) + h.proxy.recordWithheldPyPI(packageName, version, publishedAt) } delete(releases, version) continue @@ -687,7 +687,8 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( } // storeUploadTime records a release's newest upload time under the version -// PURL the file is cached under, so storedCooldown finds it. +// PURL the file is cached under, so storedCooldown finds it. The time only +// moves forward: a request that read an older document leaves a newer one. func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) { if h.proxy.DB == nil { return @@ -696,7 +697,7 @@ func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Tim if err != nil { return } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { + if err := h.proxy.DB.AdvanceVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { h.proxy.Logger.Warn("cooldown: could not store pypi upload time", "package", name, "version", version, "error", err) } @@ -757,7 +758,7 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL if h.proxy.versionDenied("pypi", name, version) { metadata["urls"] = []any{} } else if held, publishedAt := h.shouldFilterRelease(packagePURL, urls); held { - h.proxy.recordWithheldNewest("pypi", name, version, publishedAt) + h.proxy.recordWithheldPyPI(name, version, publishedAt) metadata["urls"] = []any{} } From 1669319d265cbbd72a857041c78dd735744ea688 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 18:30:40 +0100 Subject: [PATCH 6/7] Decide a PyPI download's cooldown with the newer stored upload time A download that read an upload time out of cooldown let the file through even when a newer time inside cooldown was stored meanwhile. storeUploadTime now returns the newer of the two, and versionInCooldown decides with it. --- internal/handler/cooldown_withheld_test.go | 18 ++++++++++++++++++ internal/handler/pypi.go | 20 ++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go index 362780e4..55edaab2 100644 --- a/internal/handler/cooldown_withheld_test.go +++ b/internal/handler/cooldown_withheld_test.go @@ -387,6 +387,24 @@ func TestPyPIDownloadKeepsTheNewerStoredUploadTime(t *testing.T) { assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", newer) } +// A download that read an upload time out of cooldown is still refused when +// a newer time, inside it, was stored meanwhile. +func TestPyPIDownloadDecidesWithTheNewerStoredUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + older, newer := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", + `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.recordWithheldPyPI("ruff", "2.0.0", newer) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown { + t.Errorf("versionInCooldown = %v, %v; want true from the newer stored time", inCooldown, err) + } +} + // The version JSON names one release, in urls, and records it when it is // held back. func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index accd786a..3580140a 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -668,7 +668,8 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) (bool, // // A release whose upload time cannot be determined is allowed through, matching // how fetchFilteredVersions treats it. Failing to read the times is an error. -// The time read is stored for storedCooldown. +// The time read is stored for storedCooldown, and a newer one stored +// meanwhile decides. func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) { if !h.proxy.cooldownApplies("pypi", name) { return false, nil @@ -682,25 +683,32 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( if publishedAt.IsZero() { return false, nil } - h.storeUploadTime(name, version, publishedAt) + publishedAt = h.storeUploadTime(name, version, publishedAt) return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } // storeUploadTime records a release's newest upload time under the version // PURL the file is cached under, so storedCooldown finds it. The time only -// moves forward: a request that read an older document leaves a newer one. -func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) { +// moves forward: a request that read an older document leaves a newer one, +// and gets that newer time back. +func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) time.Time { if h.proxy.DB == nil { - return + return publishedAt } packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) if err != nil { - return + return publishedAt } if err := h.proxy.DB.AdvanceVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { h.proxy.Logger.Warn("cooldown: could not store pypi upload time", "package", name, "version", version, "error", err) + return publishedAt + } + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && + ver.PublishedAt.Valid && ver.PublishedAt.Time.After(publishedAt) { + return ver.PublishedAt.Time } + return publishedAt } // storedCooldown answers the cooldown check for a download from the upload From da141d6ad82d77e53193b6ba04cd45b46a0b452b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 15 Nov 2026 21:05:18 +0100 Subject: [PATCH 7/7] Extract portable Composer metadata transformations Move the metadata rewrite (cooldown filtering, dist URLs, pushed versions, the minified field expansion), the dist lookup for downloads and the dist URL helpers out of internal/handler into internal/metadata/composer. The package takes its inputs explicitly (proxy URL, cooldown policy, evaluation time, pushed versions, a routed-package check) and reports versions cooldown holds back through a callback the handler logs and records. It imports no handler, database, HTTP or logging code, and builds for wasip1. Cooldown in the rewrite is checked with Evaluate at one EvaluatedAt, as in internal/metadata/npm. --- internal/handler/composer.go | 483 ++--------------- internal/handler/composer_hosted.go | 33 +- internal/handler/composer_rewrite_test.go | 2 +- internal/metadata/composer/imports_test.go | 44 ++ internal/metadata/composer/metadata.go | 545 ++++++++++++++++++++ internal/metadata/composer/metadata_test.go | 144 ++++++ 6 files changed, 776 insertions(+), 475 deletions(-) create mode 100644 internal/metadata/composer/imports_test.go create mode 100644 internal/metadata/composer/metadata.go create mode 100644 internal/metadata/composer/metadata_test.go diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6e77e315..1fccb2ac 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,19 +1,16 @@ package handler import ( - "bytes" "context" "encoding/json" "errors" "fmt" "net/http" - "net/url" "path" - "regexp" "strings" "time" - "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/metadata/composer" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -21,15 +18,10 @@ import ( const ( composerUpstream = "https://packagist.org" composerRepo = "https://repo.packagist.org" - composerMinified = "composer/2.0" - composerUnset = "__unset" - composerDevReset = "~dev" composerDevFile = "~dev" // suffix of the metadata file holding dev versions vendorPackageParts = 2 ) -var composerUnsetJSON = []byte(`"` + composerUnset + `"`) - // Archives are stored under pkg:composer/{vendor}/{package}, which purl // lowercases, so a package rule matches whatever case it is written in. The // spec also claims the "packagist" spelling git-pkgs writes into @@ -224,411 +216,53 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, } // rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the -// pushed versions in local under localPackage, in place of any upstream entry -// with the same version. Cooldown only filters upstream versions. A document -// that does not list localPackage at all is replaced by one holding only the -// pushed versions. +// pushed versions in local under localPackage; see composer.Rewrite. func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage string, local []composerLocalEntry) ([]byte, []string, error) { - if !json.Valid(body) { - return nil, nil, errors.New("composer metadata is not valid JSON") - } - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return nil, nil, err - } - minified := format == composerMinified - - var out bytes.Buffer - var kept []string - listed := false - out.Grow(len(body) + len(body)/8) - err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { - packages := m.Value(body) - if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { - return false, nil - } - return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { - versions := p.Value(packages) - if versions[0] != '[' { - return false, nil - } - packageName, err := jsonscan.Key(p.Key(packages)) - if err != nil { - return false, err - } - var pushed []composerLocalEntry - if packageName == localPackage { - pushed, listed = local, true - } - return true, h.writeVersions(out, packageName, versions, minified, &kept, pushed) - }) - }) - if err != nil { - return nil, nil, err - } - if len(local) > 0 && !listed { - document, err := localComposerDocument(localPackage, local) - return document, nil, err - } - return out.Bytes(), kept, nil -} - -// writeVersions writes one package's version list without the versions in -// cooldown and with each dist pointing at this proxy, and appends each version -// it writes to kept as its package name and version. -// -// Minified lists stay minified. Each version written carries the fields that -// differ from what the client has expanded so far, which is the upstream -// entry itself unless filtered versions were dropped before it; then it also -// carries the changes those versions made. Every version gets its own dist, -// since the proxied URL names the version. -// -// The pushed versions in local follow the upstream ones and take the place of -// upstream entries with the same version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []composerLocalEntry) error { - packagePURL := canonicalPackagePURL("composer", packageName) - upstream, written := newComposerFields(), newComposerFields() - devReset, first := false, true - _, routed := h.routes.match(packageName) - - out.WriteByte('[') - err := jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - upstream.reset() - written.reset() - devReset = true - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - upstream.reset() - written.reset() - } - if err := upstream.apply(entry, minified); err != nil { - return err - } - - version := upstream.string("version") - if pushedComposerVersion(local, version) || h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { - return nil - } - - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - if routed { - disableDefaultNotification(upstream) - } - h.writeVersion(out, packageName, version, upstream, written) - *kept = append(*kept, packageName+"\x00"+version) - return nil + return composer.Rewrite(body, composer.Options{ + ProxyURL: h.proxyURL, + Cooldown: h.proxy.cooldown(), + EvaluatedAt: time.Now(), + LocalPackage: localPackage, + LocalVersions: local, + Routed: func(packageName string) bool { + _, routed := h.routes.match(packageName) + return routed + }, + OnCooldownFilter: h.cooldownFiltered, }) - if err != nil { - return err - } - for _, e := range local { - pushed := newComposerFields() - if err := pushed.apply(e.entry, false); err != nil { - return err - } - if !minified { - written.reset() - } - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - h.writeVersion(out, packageName, e.version, pushed, written) - *kept = append(*kept, packageName+"\x00"+e.version) - } - out.WriteByte(']') - return nil } -// writeVersion writes the fields of upstream that differ from written, with -// dist pointing at this proxy, and "__unset" for those written has and -// upstream lacks. written is then what the client expands this version to. -func (h *ComposerHandler) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *composerFields) { - out.WriteByte('{') - first := true - writeMember := func(key, value []byte) { - if !first { - out.WriteByte(',') - } - first = false - out.Write(key) - out.WriteByte(':') - out.Write(value) +// cooldownFiltered logs a version a metadata rewrite held back and records +// it for the package page. +func (h *ComposerHandler) cooldownFiltered(packageName, version string, publishedAt time.Time) { + h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) + // A routed package's times are the route's, and aren't stored under the + // default registry's PURLs (see releaseTimeStored). + if _, routed := h.routes.match(packageName); !routed { + h.proxy.recordWithheld("composer", packageName, version, publishedAt) } - - for _, f := range upstream.fields { - value := f.value - if value == nil { - continue - } - if f.name == "dist" { - value = h.rewriteDist(packageName, version, value) - } - if bytes.Equal(value, written.get(f.name)) { - continue - } - writeMember(f.key, value) - written.set(f.name, f.key, value) - } - for _, f := range written.fields { - if f.value != nil && upstream.get(f.name) == nil { - writeMember(f.key, composerUnsetJSON) - written.set(f.name, f.key, nil) - } - } - out.WriteByte('}') } -// shouldFilterVersion returns true if the version should be excluded due to cooldown. +// shouldFilterVersion reports whether cooldown holds a version back from a +// download. func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, published []byte) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - - publishedAt, ok := composerPublishTime(published) - if !ok { + publishedAt, ok := composer.PublishTime(published) + if !ok || h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { return false } - - if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { - h.proxy.Logger.Info("cooldown: filtering composer version", - "package", packageName, "version", version) - // A routed package's times are the route's, and aren't stored under - // the default registry's PURLs (see releaseTimeStored). - if _, routed := h.routes.match(packageName); !routed { - h.proxy.recordWithheld("composer", packageName, version, publishedAt) - } - return true - } - - return false -} - -// composerPublishTime parses the raw "time" field of a version, reporting -// false when it is missing or not an RFC 3339 string. -func composerPublishTime(published []byte) (time.Time, bool) { - var timeStr string - if err := json.Unmarshal(published, &timeStr); err != nil { - return time.Time{}, false - } - publishedAt, err := time.Parse(time.RFC3339, timeStr) - return publishedAt, err == nil -} - -// disableDefaultNotification stops Composer from reporting installs of a routed -// package to the default repository's notify-batch URL, which would disclose -// the package name to it. Composer only falls back to notify-batch when a -// version has no notification-url of its own, and skips an empty one. Setting -// it on the expanded fields keeps it set for the versions that inherit them. -// Composer reads a null notification-url as absent too. -func disableDefaultNotification(fields *composerFields) { - if v := fields.get("notification-url"); v == nil || string(v) == "null" { - fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) - } -} - -// rewriteDist returns a version's dist object with its url pointing at this -// proxy. A dist without a string url is returned unchanged. -func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { - urlMember, ok, err := jsonscan.FindMember(dist, "url") - if err != nil || !ok || dist[urlMember.ValStart] != '"' { - return dist - } - var upstreamURL string - if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { - return dist - } - distType, _, _ := jsonscan.LookupString(dist, "type") - - proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) - if !ok { - return dist - } - encoded, err := json.Marshal(proxyURL) - if err != nil { - return dist - } - rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) - rewritten = append(rewritten, dist[:urlMember.ValStart]...) - rewritten = append(rewritten, encoded...) - return append(rewritten, dist[urlMember.ValEnd:]...) + h.cooldownFiltered(packageName, version, publishedAt) + return true } -// proxyDistURL returns the URL this proxy serves a version's archive at, or -// false when the upstream URL or package name gives nothing to build it from. func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distType string) (string, bool) { - if upstreamURL == "" { - return "", false - } - parts := strings.SplitN(packageName, "/", vendorPackageParts) - if len(parts) != vendorPackageParts { - return "", false - } - return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], url.PathEscape(version), distFilename(upstreamURL, distType)), true -} - -// distFilename returns the file name this proxy serves an archive under: the -// last segment of its upstream URL. -func distFilename(upstreamURL, distType string) string { - filename := "package.zip" - if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { - filename = upstreamURL[idx+1:] - } - - // GitHub zipball URLs end with a bare commit hash (no extension). - // Append .zip so the archives library can detect the format. - if path.Ext(filename) == "" && distType == "zip" { - filename += ".zip" - } - return filename + return composer.ProxyDistURL(h.proxyURL, packageName, version, upstreamURL, distType) } -// composerCommitPattern matches a full git commit hash, SHA-1 or SHA-256. -var composerCommitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) - -// upstreamDistURL returns the upstream URL of the archive served as filename, -// given the dist URL the version's metadata lists now. A branch version only -// lists its current head while lock files pin older commits, so for a branch -// a dist URL ending in a commit hash is rebuilt for the requested commit. A -// tagged release has to ask for the listed name. Any other name gives false: -// fetching the listed archive would cache it under a name it does not have. func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { - listed := distFilename(listedURL, distType) - // Proxy versions before the .zip suffix handed out bare commit names, - // and lock files written back then still ask for them. - if bare := distFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { - filename += listed[len(bare):] - } - if filename == listed { - return listedURL, true - } - if commit, ok := shaQueryCommit(listed); ok { - requested, ok := shaQueryCommit(filename) - listedSha, requestedSha := "sha="+commit, "sha="+requested - idx := strings.LastIndex(listed, listedSha) - if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { - return "", false - } - idx = strings.LastIndex(listedURL, listedSha) - return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true - } - ext := path.Ext(listed) - commit := strings.TrimSuffix(listed, ext) - requested, ok := strings.CutSuffix(filename, ext) - if !branch || !ok || !composerCommitPattern.MatchString(commit) || !composerCommitPattern.MatchString(requested) { - return "", false - } - idx := strings.LastIndex(listedURL, commit) - return listedURL[:idx] + requested + listedURL[idx+len(commit):], true -} - -// shaQueryCommit returns the commit an archive name passes in a sha query -// parameter. GitLab names the archive of every commit archive.zip and picks -// the commit that way. -func shaQueryCommit(filename string) (string, bool) { - _, query, ok := strings.Cut(filename, "?") - if !ok { - return "", false - } - values, err := url.ParseQuery(query) - if err != nil { - return "", false - } - commit := values.Get("sha") - return commit, composerCommitPattern.MatchString(commit) -} - -// composerFields holds one version's fields as Composer sees them after -// expanding the minified format: raw values under their decoded names, in the -// order the names first appeared. A removed field keeps its slot with a nil -// value, so the order stays stable when it comes back. -type composerFields struct { - fields []composerField - index map[string]int -} - -type composerField struct { - name string - key, value []byte -} - -func newComposerFields() *composerFields { - return &composerFields{index: map[string]int{}} -} - -func (f *composerFields) reset() { - f.fields = f.fields[:0] - clear(f.index) -} - -// get returns the raw value of the named field, or nil when it is not set. -func (f *composerFields) get(name string) []byte { - if i, ok := f.index[name]; ok { - return f.fields[i].value - } - return nil -} - -// string returns the named field when it is a string, or "". -func (f *composerFields) string(name string) string { - raw := f.get(name) - if len(raw) == 0 || raw[0] != '"' { - return "" - } - var s string - _ = json.Unmarshal(raw, &s) - return s -} - -// set stores value under name, written with the quoted key. A nil value -// removes the field. -func (f *composerFields) set(name string, key, value []byte) { - if i, ok := f.index[name]; ok { - f.fields[i].key, f.fields[i].value = key, value - return - } - f.index[name] = len(f.fields) - f.fields = append(f.fields, composerField{name: name, key: key, value: value}) -} - -// apply overlays the members of a version entry. In minified metadata the -// value "__unset" removes the field instead. -func (f *composerFields) apply(entry []byte, minified bool) error { - return jsonscan.EachMember(entry, func(m jsonscan.Member) error { - key, value := m.Key(entry), m.Value(entry) - if minified && jsonscan.StringIs(value, composerUnset) { - value = nil - } - if bytes.IndexByte(key, '\\') < 0 { - if i, ok := f.index[string(key[1:len(key)-1])]; ok { - f.fields[i].key, f.fields[i].value = key, value - return nil - } - } - name, err := jsonscan.Key(key) - if err != nil { - return err - } - f.set(name, key, value) - return nil - }) + return composer.UpstreamDistURL(listedURL, distType, filename, branch) } // handleDownload serves a package file, fetching and caching from upstream if needed. @@ -690,13 +324,13 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) // the name they are cached under. Proxy versions before that cached every // commit under the shared name, which is dropped once a commit is fetched. var sharedFilename string - if commit, ok := shaQueryCommit(requested); ok { + if commit, ok := composer.ShaQueryCommit(requested); ok { sharedFilename = h.routes.cacheFilename(packageName, filename) ext := path.Ext(filename) filename = strings.TrimSuffix(filename, ext) + "-" + commit + ext } cacheFilename := h.routes.cacheFilename(packageName, filename) - if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composerCommitPattern.MatchString(commit) { + if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composer.IsCommit(commit) { h.discardOtherCommit(packageName, version, cacheFilename, commit) } result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, @@ -775,7 +409,7 @@ func (h *ComposerHandler) recordPublishTime(packageName, version string, publish if !h.releaseTimeStored(packageName, version) { return } - publishedAt, ok := composerPublishTime(published) + publishedAt, ok := composer.PublishTime(published) if !ok { return } @@ -971,55 +605,10 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU return dist, nil } -// composerDistURL returns the upstream dist of version from Composer -// metadata, with an empty URL when the package has no such version or it has -// no dist URL. It walks the version list in place, expanding only as much of -// the minified format as it needs, since it runs on each archive the proxy -// has not cached yet. +// composerDistURL returns the upstream dist of version; see composer.FindDist. func composerDistURL(body []byte, packageName, version string) (composerDist, error) { - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return composerDist{}, err - } - versions, err := jsonscan.Lookup(body, "packages", packageName) - if err != nil || len(versions) == 0 || versions[0] != '[' { - return composerDist{}, err - } - minified := format == composerMinified - - fields := newComposerFields() - var dist composerDist - err = jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - fields.reset() - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - fields.reset() - } - if err := fields.apply(entry, minified); err != nil { - return err - } - if !jsonscan.StringIs(fields.get("version"), version) { - return nil - } - distRaw := fields.get("dist") - dist.url, _, _ = jsonscan.LookupString(distRaw, "url") - if dist.url != "" { - dist.distType, _, _ = jsonscan.LookupString(distRaw, "type") - dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") - dist.published = fields.get("time") - return jsonscan.ErrStop - } - return nil - }) - if errors.Is(err, jsonscan.ErrStop) { - err = nil - } - return dist, err + d, err := composer.FindDist(body, packageName, version) + return composerDist{url: d.URL, distType: d.Type, shasum: d.Shasum, published: d.Published}, err } // composerIntegrity returns the integrity to verify an archive of version @@ -1053,7 +642,7 @@ func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename case strings.ContainsAny(distURL, "?#"): return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) } - return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil + return composer.DistFilename(distURL, distType), composerIntegrity(shasum, version, true), nil } // RecordComposerPublishTime stores the publish time of a tagged release from diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index be8a1d6d..0142b344 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -6,8 +6,9 @@ import ( "fmt" "net/http" "net/url" - "strings" "time" + + "github.com/git-pkgs/proxy/internal/metadata/composer" ) // composerRootOnlyKeys are composer.json keys that only apply to the root @@ -15,10 +16,7 @@ import ( var composerRootOnlyKeys = []string{"repositories", "config", "minimum-stability", "prefer-stable"} // composerLocalEntry is a pushed version as Composer metadata lists it. -type composerLocalEntry struct { - version string - entry json.RawMessage -} +type composerLocalEntry = composer.LocalVersion // composerLocalEntries returns the pushed versions of a package for one of // its metadata files: the dev versions for the ~dev file, the tagged ones for @@ -59,7 +57,7 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] if err != nil { return nil, err } - entries = append(entries, composerLocalEntry{version: lv.Version, entry: entry}) + entries = append(entries, composerLocalEntry{Version: lv.Version, Entry: entry}) } return entries, nil } @@ -68,17 +66,6 @@ func (h *ComposerHandler) localDistURL(packageName, version, filename string) st return fmt.Sprintf("%s/composer/files/%s/%s/%s", h.proxyURL, packageName, url.PathEscape(version), filename) } -// pushedComposerVersion reports whether version is one of the pushed ones. -// Tags are listed with and without a leading v, and both name one version. -func pushedComposerVersion(local []composerLocalEntry, version string) bool { - for _, e := range local { - if strings.TrimPrefix(e.version, "v") == strings.TrimPrefix(version, "v") { - return true - } - } - return false -} - // serveMetadataWithLocal serves a package's metadata with its pushed versions // added, bypassing the rewrite cache like the npm handler does. When the // upstream does not know the package, the document holds only the pushed @@ -89,7 +76,7 @@ func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http. var err error switch { case errors.Is(upstream.err, ErrUpstreamNotFound): - out, err = localComposerDocument(packageName, local) + out, err = composer.LocalDocument(packageName, local) case upstream.err != nil: if writeUpstreamHeld(w, upstream.err) { return @@ -116,7 +103,7 @@ func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http http.Error(w, "not found", http.StatusNotFound) return } - out, err := localComposerDocument(packageName, local) + out, err := composer.LocalDocument(packageName, local) if err != nil { h.proxy.Logger.Error("failed to build composer metadata from pushed versions", "package", packageName, "error", err) http.Error(w, "failed to build package metadata", http.StatusInternalServerError) @@ -124,11 +111,3 @@ func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http } serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) } - -func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { - entries := make([]json.RawMessage, 0, len(local)) - for _, e := range local { - entries = append(entries, e.entry) - } - return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) -} diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go index 460e38e0..4934389d 100644 --- a/internal/handler/composer_rewrite_test.go +++ b/internal/handler/composer_rewrite_test.go @@ -42,7 +42,7 @@ func composerVersions(t *testing.T, output []byte, packageName string) []map[str } expanded = maps.Clone(expanded) for k, v := range fields { - if v == composerUnset { + if v == "__unset" { delete(expanded, k) continue } diff --git a/internal/metadata/composer/imports_test.go b/internal/metadata/composer/imports_test.go new file mode 100644 index 00000000..0a10bb5c --- /dev/null +++ b/internal/metadata/composer/imports_test.go @@ -0,0 +1,44 @@ +package composer + +import ( + "os/exec" + "slices" + "strings" + "testing" +) + +// TestPortableImports keeps the package free of the proxy's infrastructure: +// no handler, database, storage, metrics, HTTP or logging, so it builds for +// targets such as WASI where those are not available. +func TestPortableImports(t *testing.T) { + goTool, err := exec.LookPath("go") + if err != nil { + t.Skip("go tool not found") + } + out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + allowed := []string{ + "github.com/git-pkgs/cooldown", + "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/metadata/composer", + "github.com/git-pkgs/proxy/internal/packageurl", + "github.com/git-pkgs/purl", + "github.com/git-pkgs/vers", + "github.com/package-url/packageurl-go", + } + forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"} + for line := range strings.Lines(string(out)) { + standard, path, _ := strings.Cut(strings.TrimSpace(line), " ") + if standard == "true" { + if slices.Contains(forbiddenStd, path) { + t.Errorf("imports %s", path) + } + continue + } + if !slices.Contains(allowed, path) { + t.Errorf("imports %s, which is not on the portable list", path) + } + } +} diff --git a/internal/metadata/composer/metadata.go b/internal/metadata/composer/metadata.go new file mode 100644 index 00000000..55e35fc6 --- /dev/null +++ b/internal/metadata/composer/metadata.go @@ -0,0 +1,545 @@ +// Package composer transforms Composer repository metadata without storage or +// network access. +package composer + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/url" + "path" + "regexp" + "strings" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +const ( + // Minified is the "minified" value of Composer 2 metadata, whose + // version lists only carry the fields that change. + Minified = "composer/2.0" + // DevReset starts over the expanded fields in a minified list, before + // the dev versions. + DevReset = "~dev" + unset = "__unset" + vendorPackageParts = 2 +) + +var unsetJSON = []byte(`"` + unset + `"`) + +// CooldownPolicy evaluates version age at the caller's supplied time. +type CooldownPolicy interface { + Enabled() bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision +} + +// LocalVersion is a pushed version as Composer metadata lists it. +type LocalVersion struct { + Version string + Entry json.RawMessage +} + +// Options supplies the public URL, policy and pushed versions for a rewrite. +// EvaluatedAt is used as given; the transformation never reads the clock. +type Options struct { + ProxyURL string + Cooldown CooldownPolicy + EvaluatedAt time.Time + // LocalPackage and LocalVersions are the pushed versions to list. + LocalPackage string + LocalVersions []LocalVersion + // Routed reports whether a package is served from a route rather than + // the default repository; its versions don't report installs to the + // default repository. + Routed func(packageName string) bool + // OnCooldownFilter is called synchronously for each version cooldown + // holds back, with its publish time. + OnCooldownFilter func(packageName, version string, publishedAt time.Time) +} + +// Dist is the upstream archive of a version: its URL, type and shasum, and +// the raw publish time the cooldown is checked against. +type Dist struct { + URL, Type, Shasum string + Published []byte +} + +// Rewrite filters the versions cooldown holds back and points each dist at +// the proxy, returning the kept package names and versions without expanding +// minified lists. The pushed versions in o.LocalVersions are listed under +// o.LocalPackage in place of any upstream entry with the same version; +// cooldown only filters upstream versions. A document that does not list +// o.LocalPackage at all is replaced by one holding only the pushed versions. +func Rewrite(body []byte, o Options) ([]byte, []string, error) { + localPackage, local := o.LocalPackage, o.LocalVersions + if !json.Valid(body) { + return nil, nil, errors.New("composer metadata is not valid JSON") + } + format, _, err := jsonscan.LookupString(body, "minified") + if err != nil { + return nil, nil, err + } + minified := format == Minified + + var out bytes.Buffer + var kept []string + listed := false + out.Grow(len(body) + len(body)/8) + err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { + packages := m.Value(body) + if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { + return false, nil + } + return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { + versions := p.Value(packages) + if versions[0] != '[' { + return false, nil + } + packageName, err := jsonscan.Key(p.Key(packages)) + if err != nil { + return false, err + } + var pushed []LocalVersion + if packageName == localPackage { + pushed, listed = local, true + } + return true, o.writeVersions(out, packageName, versions, minified, &kept, pushed) + }) + }) + if err != nil { + return nil, nil, err + } + if len(local) > 0 && !listed { + document, err := LocalDocument(localPackage, local) + return document, nil, err + } + return out.Bytes(), kept, nil +} + +// writeVersions writes one package's version list without the versions in +// cooldown and with each dist pointing at this proxy, and appends each version +// it writes to kept as its package name and version. +// +// Minified lists stay minified. Each version written carries the fields that +// differ from what the client has expanded so far, which is the upstream +// entry itself unless filtered versions were dropped before it; then it also +// carries the changes those versions made. Every version gets its own dist, +// since the proxied URL names the version. +// +// The pushed versions in local follow the upstream ones and take the place of +// upstream entries with the same version. +func (o Options) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []LocalVersion) error { + packagePURL := packageurl.MakeString("composer", packageName, "") + upstream, written := newFields(), newFields() + devReset, first := false, true + routed := o.Routed != nil && o.Routed(packageName) + + out.WriteByte('[') + err := jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, DevReset) { + upstream.reset() + written.reset() + devReset = true + return nil + } + if entry[0] != '{' { + return nil + } + if !minified { + upstream.reset() + written.reset() + } + if err := upstream.apply(entry, minified); err != nil { + return err + } + + version := upstream.string("version") + if PushedVersion(local, version) || o.heldBack(packagePURL, packageName, version, upstream.get("time")) { + return nil + } + + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + DevReset + `",`) + devReset = false + } + if routed { + disableDefaultNotification(upstream) + } + o.writeVersion(out, packageName, version, upstream, written) + *kept = append(*kept, packageName+"\x00"+version) + return nil + }) + if err != nil { + return err + } + for _, e := range local { + pushed := newFields() + if err := pushed.apply(e.Entry, false); err != nil { + return err + } + if !minified { + written.reset() + } + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + DevReset + `",`) + devReset = false + } + o.writeVersion(out, packageName, e.Version, pushed, written) + *kept = append(*kept, packageName+"\x00"+e.Version) + } + out.WriteByte(']') + return nil +} + +// writeVersion writes the fields of upstream that differ from written, with +// dist pointing at this proxy, and "__unset" for those written has and +// upstream lacks. written is then what the client expands this version to. +func (o Options) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *fields) { + out.WriteByte('{') + first := true + writeMember := func(key, value []byte) { + if !first { + out.WriteByte(',') + } + first = false + out.Write(key) + out.WriteByte(':') + out.Write(value) + } + + for _, f := range upstream.fields { + value := f.value + if value == nil { + continue + } + if f.name == "dist" { + value = o.rewriteDist(packageName, version, value) + } + if bytes.Equal(value, written.get(f.name)) { + continue + } + writeMember(f.key, value) + written.set(f.name, f.key, value) + } + for _, f := range written.fields { + if f.value != nil && upstream.get(f.name) == nil { + writeMember(f.key, unsetJSON) + written.set(f.name, f.key, nil) + } + } + out.WriteByte('}') +} + +// heldBack reports whether cooldown holds a version back. +func (o Options) heldBack(packagePURL, packageName, version string, published []byte) bool { + if o.Cooldown == nil || !o.Cooldown.Enabled() { + return false + } + publishedAt, ok := PublishTime(published) + if !ok || o.Cooldown.Evaluate("composer", packagePURL, publishedAt, o.EvaluatedAt).Allowed { + return false + } + if o.OnCooldownFilter != nil { + o.OnCooldownFilter(packageName, version, publishedAt) + } + return true +} + +// PublishTime parses the raw "time" field of a version, reporting false when +// it is missing or not an RFC 3339 string. +func PublishTime(published []byte) (time.Time, bool) { + var timeStr string + if err := json.Unmarshal(published, &timeStr); err != nil { + return time.Time{}, false + } + publishedAt, err := time.Parse(time.RFC3339, timeStr) + return publishedAt, err == nil +} + +// disableDefaultNotification stops Composer from reporting installs of a routed +// package to the default repository's notify-batch URL, which would disclose +// the package name to it. Composer only falls back to notify-batch when a +// version has no notification-url of its own, and skips an empty one. Setting +// it on the expanded fields keeps it set for the versions that inherit them. +// Composer reads a null notification-url as absent too. +func disableDefaultNotification(fields *fields) { + if v := fields.get("notification-url"); v == nil || string(v) == "null" { + fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) + } +} + +// rewriteDist returns a version's dist object with its url pointing at this +// proxy. A dist without a string url is returned unchanged. +func (o Options) rewriteDist(packageName, version string, dist []byte) []byte { + urlMember, ok, err := jsonscan.FindMember(dist, "url") + if err != nil || !ok || dist[urlMember.ValStart] != '"' { + return dist + } + var upstreamURL string + if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { + return dist + } + distType, _, _ := jsonscan.LookupString(dist, "type") + + proxyURL, ok := ProxyDistURL(o.ProxyURL, packageName, version, upstreamURL, distType) + if !ok { + return dist + } + encoded, err := json.Marshal(proxyURL) + if err != nil { + return dist + } + rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) + rewritten = append(rewritten, dist[:urlMember.ValStart]...) + rewritten = append(rewritten, encoded...) + return append(rewritten, dist[urlMember.ValEnd:]...) +} + +// ProxyDistURL returns the URL the proxy at proxyURL serves a version's +// archive at, or false when the upstream URL or package name gives nothing to +// build it from. +func ProxyDistURL(proxyURL, packageName, version, upstreamURL, distType string) (string, bool) { + if upstreamURL == "" { + return "", false + } + parts := strings.SplitN(packageName, "/", vendorPackageParts) + if len(parts) != vendorPackageParts { + return "", false + } + return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", + proxyURL, parts[0], parts[1], url.PathEscape(version), DistFilename(upstreamURL, distType)), true +} + +// DistFilename returns the file name the proxy serves an archive under: the +// last segment of its upstream URL. +func DistFilename(upstreamURL, distType string) string { + filename := "package.zip" + if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { + filename = upstreamURL[idx+1:] + } + + // GitHub zipball URLs end with a bare commit hash (no extension). + // Append .zip so the archives library can detect the format. + if path.Ext(filename) == "" && distType == "zip" { + filename += ".zip" + } + return filename +} + +// commitPattern matches a full git commit hash, SHA-1 or SHA-256. +var commitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) + +// IsCommit reports whether s is a full git commit hash, SHA-1 or SHA-256. +func IsCommit(s string) bool { + return commitPattern.MatchString(s) +} + +// UpstreamDistURL returns the upstream URL of the archive served as filename, +// given the dist URL the version's metadata lists now. A branch version only +// lists its current head while lock files pin older commits, so for a branch +// a dist URL ending in a commit hash is rebuilt for the requested commit. A +// tagged release has to ask for the listed name. Any other name gives false: +// fetching the listed archive would cache it under a name it does not have. +func UpstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { + listed := DistFilename(listedURL, distType) + // Proxy versions before the .zip suffix handed out bare commit names, + // and lock files written back then still ask for them. + if bare := DistFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { + filename += listed[len(bare):] + } + if filename == listed { + return listedURL, true + } + if commit, ok := ShaQueryCommit(listed); ok { + requested, ok := ShaQueryCommit(filename) + listedSha, requestedSha := "sha="+commit, "sha="+requested + idx := strings.LastIndex(listed, listedSha) + if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { + return "", false + } + idx = strings.LastIndex(listedURL, listedSha) + return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true + } + ext := path.Ext(listed) + commit := strings.TrimSuffix(listed, ext) + requested, ok := strings.CutSuffix(filename, ext) + if !branch || !ok || !commitPattern.MatchString(commit) || !commitPattern.MatchString(requested) { + return "", false + } + idx := strings.LastIndex(listedURL, commit) + return listedURL[:idx] + requested + listedURL[idx+len(commit):], true +} + +// ShaQueryCommit returns the commit an archive name passes in a sha query +// parameter. GitLab names the archive of every commit archive.zip and picks +// the commit that way. +func ShaQueryCommit(filename string) (string, bool) { + _, query, ok := strings.Cut(filename, "?") + if !ok { + return "", false + } + values, err := url.ParseQuery(query) + if err != nil { + return "", false + } + commit := values.Get("sha") + return commit, commitPattern.MatchString(commit) +} + +// FindDist returns the upstream dist of version from Composer metadata, with an empty URL when the package has no such version or it has +// no dist URL. It walks the version list in place, expanding only as much of +// the minified format as it needs, since it runs on each archive the proxy +// has not cached yet. +func FindDist(body []byte, packageName, version string) (Dist, error) { + format, _, err := jsonscan.LookupString(body, "minified") + if err != nil { + return Dist{}, err + } + versions, err := jsonscan.Lookup(body, "packages", packageName) + if err != nil || len(versions) == 0 || versions[0] != '[' { + return Dist{}, err + } + minified := format == Minified + + fields := newFields() + var dist Dist + err = jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, DevReset) { + fields.reset() + return nil + } + if entry[0] != '{' { + return nil + } + if !minified { + fields.reset() + } + if err := fields.apply(entry, minified); err != nil { + return err + } + if !jsonscan.StringIs(fields.get("version"), version) { + return nil + } + distRaw := fields.get("dist") + dist.URL, _, _ = jsonscan.LookupString(distRaw, "url") + if dist.URL != "" { + dist.Type, _, _ = jsonscan.LookupString(distRaw, "type") + dist.Shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") + dist.Published = fields.get("time") + return jsonscan.ErrStop + } + return nil + }) + if errors.Is(err, jsonscan.ErrStop) { + err = nil + } + return dist, err +} + +// PushedVersion reports whether version is one of the pushed ones. Tags are +// listed with and without a leading v, and both name one version. +func PushedVersion(local []LocalVersion, version string) bool { + for _, e := range local { + if strings.TrimPrefix(e.Version, "v") == strings.TrimPrefix(version, "v") { + return true + } + } + return false +} + +// LocalDocument is a metadata document listing only the pushed versions. +func LocalDocument(packageName string, local []LocalVersion) ([]byte, error) { + entries := make([]json.RawMessage, 0, len(local)) + for _, e := range local { + entries = append(entries, e.Entry) + } + return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) +} + +// fields holds one version's fields as Composer sees them after +// expanding the minified format: raw values under their decoded names, in the +// order the names first appeared. A removed field keeps its slot with a nil +// value, so the order stays stable when it comes back. +type fields struct { + fields []field + index map[string]int +} + +type field struct { + name string + key, value []byte +} + +func newFields() *fields { + return &fields{index: map[string]int{}} +} + +func (f *fields) reset() { + f.fields = f.fields[:0] + clear(f.index) +} + +// get returns the raw value of the named field, or nil when it is not set. +func (f *fields) get(name string) []byte { + if i, ok := f.index[name]; ok { + return f.fields[i].value + } + return nil +} + +// string returns the named field when it is a string, or "". +func (f *fields) string(name string) string { + raw := f.get(name) + if len(raw) == 0 || raw[0] != '"' { + return "" + } + var s string + _ = json.Unmarshal(raw, &s) + return s +} + +// set stores value under name, written with the quoted key. A nil value +// removes the field. +func (f *fields) set(name string, key, value []byte) { + if i, ok := f.index[name]; ok { + f.fields[i].key, f.fields[i].value = key, value + return + } + f.index[name] = len(f.fields) + f.fields = append(f.fields, field{name: name, key: key, value: value}) +} + +// apply overlays the members of a version entry. In minified metadata the +// value "__unset" removes the field instead. +func (f *fields) apply(entry []byte, minified bool) error { + return jsonscan.EachMember(entry, func(m jsonscan.Member) error { + key, value := m.Key(entry), m.Value(entry) + if minified && jsonscan.StringIs(value, unset) { + value = nil + } + if bytes.IndexByte(key, '\\') < 0 { + if i, ok := f.index[string(key[1:len(key)-1])]; ok { + f.fields[i].key, f.fields[i].value = key, value + return nil + } + } + name, err := jsonscan.Key(key) + if err != nil { + return err + } + f.set(name, key, value) + return nil + }) +} diff --git a/internal/metadata/composer/metadata_test.go b/internal/metadata/composer/metadata_test.go new file mode 100644 index 00000000..5bc561b5 --- /dev/null +++ b/internal/metadata/composer/metadata_test.go @@ -0,0 +1,144 @@ +package composer + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// dayCooldown holds back versions published within a day of the time a +// rewrite is evaluated at, so tests can tell that time from the clock. +type dayCooldown struct{ enabled bool } + +func (c dayCooldown) Enabled() bool { return c.enabled } + +func (dayCooldown) Evaluate(_, _ string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + return cooldown.Decision{Allowed: evaluatedAt.Sub(publishedAt) >= 24*time.Hour} +} + +// A minified document: the second entry inherits the first one's fields. +const minifiedDocument = `{"minified": "composer/2.0", "packages": {"acme/tool": [ + {"name": "acme/tool", "version": "2.0.0", "time": "2026-01-10T12:00:00+00:00", + "dist": {"type": "zip", "url": "https://codeload.example/acme/tool/zip/bbb", "shasum": ""}}, + {"version": "1.0.0", "time": "2026-01-01T00:00:00+00:00", + "dist": {"type": "zip", "url": "https://codeload.example/acme/tool/zip/aaa", "shasum": ""}} +]}}` + +func testOptions() Options { + return Options{ + ProxyURL: "https://proxy.example", + Cooldown: dayCooldown{enabled: true}, + EvaluatedAt: time.Date(2026, 1, 11, 0, 0, 0, 0, time.UTC), + } +} + +// Cooldown is decided at EvaluatedAt and reported; the version that stays +// still carries the fields it inherited from the one left out. +func TestRewrite(t *testing.T) { + o := testOptions() + var filtered []string + o.OnCooldownFilter = func(name, version string, at time.Time) { + filtered = append(filtered, name+"@"+version+" "+at.UTC().Format(time.RFC3339)) + } + out, kept, err := Rewrite([]byte(minifiedDocument), o) + if err != nil { + t.Fatalf("Rewrite: %v", err) + } + if !slices.Equal(kept, []string{"acme/tool\x001.0.0"}) { + t.Errorf("kept = %q", kept) + } + if !slices.Equal(filtered, []string{"acme/tool@2.0.0 2026-01-10T12:00:00Z"}) { + t.Errorf("filtered = %v", filtered) + } + var doc struct { + Packages map[string][]map[string]any `json:"packages"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + versions := doc.Packages["acme/tool"] + if len(versions) != 1 || versions[0]["name"] != "acme/tool" || versions[0]["version"] != "1.0.0" { + t.Fatalf("versions = %v", versions) + } + dist, _ := versions[0]["dist"].(map[string]any) + if dist["url"] != "https://proxy.example/composer/files/acme/tool/1.0.0/aaa.zip" { + t.Errorf("dist url = %v", dist["url"]) + } +} + +// A disabled or missing policy keeps every version. +func TestRewriteWithoutCooldown(t *testing.T) { + for _, policy := range []CooldownPolicy{nil, dayCooldown{enabled: false}} { + o := testOptions() + o.Cooldown = policy + if _, kept, err := Rewrite([]byte(minifiedDocument), o); err != nil || len(kept) != 2 { + t.Errorf("policy %v: kept = %q, %v; want both versions", policy, kept, err) + } + } +} + +// Pushed versions take the place of upstream ones, and a routed package's +// versions don't report installs to the default repository. +func TestRewriteLocalAndRouted(t *testing.T) { + o := testOptions() + o.Cooldown = nil + o.LocalPackage = "acme/tool" + o.LocalVersions = []LocalVersion{{Version: "v1.0.0", Entry: json.RawMessage(`{"name": "acme/tool", "version": "1.0.0", "dist": {"type": "zip", "url": "https://proxy.example/composer/files/acme/tool/1.0.0/local.zip"}}`)}} + o.Routed = func(name string) bool { return name == "acme/tool" } + out, kept, err := Rewrite([]byte(minifiedDocument), o) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(kept, []string{"acme/tool\x002.0.0", "acme/tool\x00v1.0.0"}) { + t.Errorf("kept = %q", kept) + } + if !strings.Contains(string(out), "local.zip") || strings.Contains(string(out), "zip/aaa") { + t.Errorf("the pushed 1.0.0 did not replace the upstream one: %s", out) + } + if !strings.Contains(string(out), `"notification-url":""`) { + t.Errorf("a routed package's versions keep the default notification URL: %s", out) + } + + doc, err := LocalDocument("acme/other", o.LocalVersions) + if err != nil || !strings.Contains(string(doc), `"acme/other"`) { + t.Errorf("LocalDocument = %s, %v", doc, err) + } +} + +func TestFindDist(t *testing.T) { + dist, err := FindDist([]byte(minifiedDocument), "acme/tool", "1.0.0") + if err != nil || dist.URL != "https://codeload.example/acme/tool/zip/aaa" || dist.Type != "zip" || + string(dist.Published) != `"2026-01-01T00:00:00+00:00"` { + t.Errorf("FindDist = %+v, %v", dist, err) + } + if dist, err := FindDist([]byte(minifiedDocument), "acme/tool", "9.9.9"); err != nil || dist.URL != "" { + t.Errorf("missing version = %+v, %v", dist, err) + } +} + +func TestDistURLs(t *testing.T) { + if got, ok := ProxyDistURL("https://proxy.example", "acme/tool", "1.0.0", "https://codeload.example/acme/tool/zip/aaa", "zip"); !ok || + got != "https://proxy.example/composer/files/acme/tool/1.0.0/aaa.zip" { + t.Errorf("ProxyDistURL = %q, %v", got, ok) + } + if _, ok := ProxyDistURL("https://proxy.example", "tool", "1.0.0", "https://x/y.zip", "zip"); ok { + t.Error("ProxyDistURL accepted a name without a vendor") + } + commit := strings.Repeat("a", 40) + if !IsCommit(commit) || IsCommit("main") { + t.Error("IsCommit") + } + if c, ok := ShaQueryCommit("archive.zip?sha=" + commit); !ok || c != commit { + t.Errorf("ShaQueryCommit = %q, %v", c, ok) + } + if at, ok := PublishTime([]byte(`"2026-01-01T00:00:00+00:00"`)); !ok || !at.Equal(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) { + t.Errorf("PublishTime = %v, %v", at, ok) + } + if !PushedVersion([]LocalVersion{{Version: "v2.0.0"}}, "2.0.0") || PushedVersion(nil, "2.0.0") { + t.Error("PushedVersion") + } +}