diff --git a/docs/configuration.md b/docs/configuration.md
index e7305c40..89257564 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1325,7 +1325,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma
Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families.
-The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
+The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. Neither do a Maven named repository and routed npm and Composer packages, whose publish times are kept apart from the default registry's, nor a Go module with capitals in its path, whose times are stored under the escaped path. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below.
diff --git a/internal/database/database_test.go b/internal/database/database_test.go
index 8a2e0574..398da071 100644
--- a/internal/database/database_test.go
+++ b/internal/database/database_test.go
@@ -1245,5 +1245,120 @@ func TestVersionPublishedAtIsUTC(t *testing.T) {
t.Fatal(err)
}
check("pkg:cargo/demo@2.0.0")
+
+ if err := db.RecordWithheldVersion(&Package{PURL: "pkg:cargo/demo", Ecosystem: "cargo", Name: "demo"},
+ "pkg:cargo/demo@3.0.0", written); err != nil {
+ t.Fatal(err)
+ }
+ check("pkg:cargo/demo@3.0.0")
+ })
+}
+
+// A version a cooldown filter holds back is recorded with its publish time,
+// and its package gets a row so the package page exists before anything is
+// downloaded. Rows already there keep what they have.
+func TestRecordWithheldVersion(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ const pkgPURL = "pkg:npm/leftpad"
+ published := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC)
+ pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"}
+
+ if err := db.RecordWithheldVersion(pkg, pkgPURL+"@2.0.0", published); err != nil {
+ t.Fatalf("RecordWithheldVersion: %v", err)
+ }
+ got, err := db.GetPackageByEcosystemName("npm", "leftpad")
+ if err != nil || got == nil {
+ t.Fatalf("package row = %v, %v", got, err)
+ }
+ if got.PURL != pkgPURL {
+ t.Errorf("package purl = %q, want %q", got.PURL, pkgPURL)
+ }
+ ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0")
+ if err != nil || ver == nil {
+ t.Fatalf("version row = %v, %v", ver, err)
+ }
+ if ver.PackagePURL != pkgPURL || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) {
+ t.Errorf("version = %+v, want package %s published %v", ver, pkgPURL, published)
+ }
+ })
+}
+
+// For an ecosystem whose release time is its newest file's, a later time
+// replaces the stored one and an earlier one doesn't.
+func TestRecordWithheldVersionNewest(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ const pkgPURL = "pkg:pypi/ruff"
+ first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC)
+ later := first.Add(time.Hour)
+ pkg := &Package{PURL: pkgPURL, Ecosystem: "pypi", Name: "ruff"}
+ for _, at := range []time.Time{first, later, first} {
+ if err := db.RecordWithheldVersionNewest(pkg, pkgPURL+"@2.0.0", at); err != nil {
+ t.Fatalf("RecordWithheldVersionNewest: %v", err)
+ }
+ }
+ ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0")
+ if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) {
+ t.Errorf("version = %+v, %v; want published %v", ver, err, later)
+ }
+ if got, err := db.GetPackageByEcosystemName("pypi", "ruff"); err != nil || got == nil {
+ t.Errorf("package row = %v, %v", got, err)
+ }
+ })
+}
+
+// AdvanceVersionPublishedAt only moves a stored time forward, and creates
+// the row when there is none.
+func TestAdvanceVersionPublishedAt(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ const pkgPURL = "pkg:pypi/ruff"
+ first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC)
+ later := first.Add(time.Hour)
+ for _, at := range []time.Time{first, later, first} {
+ if err := db.AdvanceVersionPublishedAt(pkgPURL+"@2.0.0", pkgPURL, at); err != nil {
+ t.Fatalf("AdvanceVersionPublishedAt: %v", err)
+ }
+ }
+ ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0")
+ if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) {
+ t.Errorf("version = %+v, %v; want published %v", ver, err, later)
+ }
+ })
+}
+
+func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) {
+ runWithBothDatabases(t, func(t *testing.T, db *DB) {
+ const pkgPURL = "pkg:npm/leftpad"
+ stored := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC)
+ other := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC)
+ if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad",
+ Description: sql.NullString{String: "pads left", Valid: true}}); err != nil {
+ t.Fatalf("UpsertPackage: %v", err)
+ }
+ if err := db.SetVersionPublishedAt(pkgPURL+"@1.0.0", pkgPURL, stored); err != nil {
+ t.Fatalf("SetVersionPublishedAt: %v", err)
+ }
+ if err := db.UpsertVersion(&Version{PURL: pkgPURL + "@3.0.0", PackagePURL: pkgPURL}); err != nil {
+ t.Fatalf("UpsertVersion: %v", err)
+ }
+
+ pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"}
+ for _, v := range []string{"1.0.0", "3.0.0"} {
+ if err := db.RecordWithheldVersion(pkg, pkgPURL+"@"+v, other); err != nil {
+ t.Fatalf("RecordWithheldVersion %s: %v", v, err)
+ }
+ }
+
+ got, err := db.GetPackageByPURL(pkgPURL)
+ if err != nil || got == nil || got.Description.String != "pads left" {
+ t.Errorf("package = %+v, %v; want the description kept", got, err)
+ }
+ ver, err := db.GetVersionByPURL(pkgPURL + "@1.0.0")
+ if err != nil || ver == nil || !ver.PublishedAt.Time.Equal(stored) {
+ t.Errorf("1.0.0 = %+v, %v; want the stored time %v kept", ver, err, stored)
+ }
+ ver, err = db.GetVersionByPURL(pkgPURL + "@3.0.0")
+ if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(other) {
+ t.Errorf("3.0.0 = %+v, %v; want the empty time filled with %v", ver, err, other)
+ }
})
}
diff --git a/internal/database/queries.go b/internal/database/queries.go
index 6f693178..10cb43d3 100644
--- a/internal/database/queries.go
+++ b/internal/database/queries.go
@@ -225,6 +225,76 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt
return nil
}
+// AdvanceVersionPublishedAt is SetVersionPublishedAt for a time that only
+// moves forward, such as a PyPI release's newest upload: an earlier time,
+// read from an older document, leaves the stored one.
+func (db *DB) AdvanceVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error {
+ now := time.Now()
+ _, err := db.Exec(db.Rebind(`
+ INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(purl) DO UPDATE SET
+ published_at = excluded.published_at,
+ updated_at = excluded.updated_at
+ WHERE versions.published_at IS NULL OR versions.published_at < excluded.published_at
+ `), versionPURL, packagePURL, publishedAt.UTC(), now, now)
+ if err != nil {
+ return fmt.Errorf("advancing version publish time: %w", err)
+ }
+ return nil
+}
+
+// RecordWithheldVersion records a version a cooldown filter held back: its
+// publish time, and a row for its package so the package page exists before
+// anything is downloaded. It only adds what is missing. An existing package
+// row is left as it is, and a stored publish time is not overwritten, so the
+// filters can call it for every version they hold back without reading
+// first.
+func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error {
+ return db.recordWithheldVersion(pkg, versionPURL, publishedAt, "versions.published_at IS NULL")
+}
+
+// RecordWithheldVersionNewest is RecordWithheldVersion for an ecosystem whose
+// release time is its newest file's, such as PyPI: a later time replaces the
+// stored one, since a file can be added to a release after it was held back.
+func (db *DB) RecordWithheldVersionNewest(pkg *Package, versionPURL string, publishedAt time.Time) error {
+ return db.recordWithheldVersion(pkg, versionPURL, publishedAt,
+ "versions.published_at IS NULL OR versions.published_at < excluded.published_at")
+}
+
+// recordWithheldVersion records a held back version, updating the stored
+// publish time where replace, a condition on the existing row, holds.
+func (db *DB) recordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time, replace string) error {
+ now := time.Now()
+ tx, err := db.Beginx()
+ if err != nil {
+ return fmt.Errorf("recording withheld version: %w", err)
+ }
+ defer func() { _ = tx.Rollback() }()
+
+ if _, err := tx.Exec(db.Rebind(`
+ INSERT INTO packages (purl, ecosystem, name, created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(purl) DO NOTHING
+ `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil {
+ return fmt.Errorf("recording withheld version's package: %w", err)
+ }
+ if _, err := tx.Exec(db.Rebind(`
+ INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(purl) DO UPDATE SET
+ published_at = excluded.published_at,
+ updated_at = excluded.updated_at
+ WHERE `+replace+`
+ `), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil {
+ return fmt.Errorf("recording withheld version: %w", err)
+ }
+ if err := tx.Commit(); err != nil {
+ return fmt.Errorf("recording withheld version: %w", err)
+ }
+ return nil
+}
+
// GetFetchedVersionPURLs returns the versions of a package that have an
// artifact row, cached now or evicted since. Versions without one were only
// seen in metadata.
diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go
index 0c742145..337c8031 100644
--- a/internal/handler/cargo.go
+++ b/internal/handler/cargo.go
@@ -180,6 +180,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr
h.proxy.Logger.Info("cooldown: filtering cargo version",
"crate", crate.Name, "version", crate.Version,
"published", crate.PublishTime)
+ h.proxy.recordWithheld("cargo", crate.Name, crate.Version, publishedAt)
continue
}
diff --git a/internal/handler/composer.go b/internal/handler/composer.go
index 6b82cfff..6e77e315 100644
--- a/internal/handler/composer.go
+++ b/internal/handler/composer.go
@@ -409,6 +409,11 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version
if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) {
h.proxy.Logger.Info("cooldown: filtering composer version",
"package", packageName, "version", version)
+ // A routed package's times are the route's, and aren't stored under
+ // the default registry's PURLs (see releaseTimeStored).
+ if _, routed := h.routes.match(packageName); !routed {
+ h.proxy.recordWithheld("composer", packageName, version, publishedAt)
+ }
return true
}
diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go
index b765f9a5..fa9120da 100644
--- a/internal/handler/cooldown_times.go
+++ b/internal/handler/cooldown_times.go
@@ -138,8 +138,8 @@ func (d timedDocuments) rewriteDigest() string {
}
// withheld reports whether cooldown leaves version out, given the
-// package's publish times.
-func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool {
+// package's publish times, and records a version it leaves out.
+func (d timedDocuments) withheld(p *Proxy, ecosystem, name, version string) bool {
published, ok := d.published[version]
if !ok {
return d.withholdUnknown
@@ -147,7 +147,11 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin
if published.IsZero() {
return false
}
- return !p.cooldown().IsAllowed(ecosystem, packagePURL, published)
+ if p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), published) {
+ return false
+ }
+ p.recordWithheld(ecosystem, name, version, published)
+ return true
}
// decodedMetadataBody returns res's body without its Content-Encoding. The
diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go
new file mode 100644
index 00000000..52c12a6a
--- /dev/null
+++ b/internal/handler/cooldown_withheld.go
@@ -0,0 +1,42 @@
+package handler
+
+import (
+ "time"
+
+ "github.com/git-pkgs/proxy/internal/database"
+)
+
+// recordWithheld stores the publish time of a version a cooldown filter held
+// back, so the package page can list it with when it becomes available,
+// also for a package nobody has downloaded yet. name is the package name as
+// the ecosystem's downloads store it, so both land on the same package row.
+// A version without a package name, a version or a known time is skipped,
+// and a failed write is only logged: the metadata is served either way.
+func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) {
+ p.recordWithheldWith(ecosystem, name, version, publishedAt, false)
+}
+
+// recordWithheldPyPI is recordWithheld for a PyPI release, whose time is its
+// newest file's: a later time replaces the stored one.
+func (p *Proxy) recordWithheldPyPI(name, version string, publishedAt time.Time) {
+ p.recordWithheldWith("pypi", name, version, publishedAt, true)
+}
+
+func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) {
+ if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() {
+ return
+ }
+ packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version)
+ if err != nil {
+ return
+ }
+ pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}
+ record := p.DB.RecordWithheldVersion
+ if newest {
+ record = p.DB.RecordWithheldVersionNewest
+ }
+ if err := record(pkg, versionPURL, publishedAt); err != nil {
+ p.Logger.Warn("cooldown: failed to record withheld version",
+ "ecosystem", ecosystem, "name", name, "version", version, "error", err)
+ }
+}
diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go
new file mode 100644
index 00000000..55edaab2
--- /dev/null
+++ b/internal/handler/cooldown_withheld_test.go
@@ -0,0 +1,554 @@
+package handler
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/cooldown"
+ "github.com/git-pkgs/proxy/internal/database"
+)
+
+// withheldTestProxy returns a proxy with a database and a three day cooldown,
+// and the times of a version old enough to pass it and one it holds back.
+func withheldTestProxy(t *testing.T) (proxy *Proxy, db *database.DB, old, recent time.Time) {
+ t.Helper()
+ proxy, db, _, _ = setupTestProxy(t)
+ proxy.Cooldown = &cooldown.Config{Default: "3d"}
+ now := time.Now().UTC().Truncate(time.Second)
+ return proxy, db, now.Add(-10 * 24 * time.Hour), now.Add(-time.Hour)
+}
+
+// assertWithheldRecorded checks that a version a filter held back is stored
+// with its publish time, under a package row the package page finds by
+// ecosystem and name.
+func assertWithheldRecorded(t *testing.T, db *database.DB, ecosystem, name, version string, published time.Time) {
+ t.Helper()
+ pkg, err := db.GetPackageByEcosystemName(ecosystem, name)
+ if err != nil || pkg == nil {
+ t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err)
+ }
+ ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version))
+ if err != nil || ver == nil {
+ t.Fatalf("version %s = %v, %v; want a row", canonicalVersionPURL(ecosystem, name, version), ver, err)
+ }
+ if ver.PackagePURL != pkg.PURL {
+ t.Errorf("version's package = %q, want %q", ver.PackagePURL, pkg.PURL)
+ }
+ if !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) {
+ t.Errorf("version %s published %v, want %v", version, ver.PublishedAt, published)
+ }
+}
+
+// assertNotRecorded checks that a version the filter let through got no row:
+// only held back versions are recorded.
+func assertNotRecorded(t *testing.T, db *database.DB, ecosystem, name, version string) {
+ t.Helper()
+ ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version))
+ if err != nil {
+ t.Fatalf("GetVersionByPURL: %v", err)
+ }
+ if ver != nil {
+ t.Errorf("version %s recorded although cooldown let it through", version)
+ }
+}
+
+func TestRecordWithheldNeedsADatabase(t *testing.T) {
+ proxy := testProxy()
+ proxy.Cooldown = &cooldown.Config{Default: "3d"}
+ proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Now())
+}
+
+func TestRecordWithheldSkipsUnknownTimes(t *testing.T) {
+ proxy, db, _, _ := withheldTestProxy(t)
+ proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Time{})
+ if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil {
+ t.Errorf("package = %v, %v; want nothing recorded without a time", pkg, err)
+ }
+}
+
+// Without a version the version PURL would be the package's own.
+func TestRecordWithheldSkipsVersionsWithoutAName(t *testing.T) {
+ proxy, db, _, recent := withheldTestProxy(t)
+ proxy.recordWithheld("npm", "leftpad", "", recent)
+ if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil {
+ t.Errorf("package = %v, %v; want nothing recorded without a version", pkg, err)
+ }
+}
+
+func TestNPMMetadataRecordsWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"}
+ input := `{"name":"testpkg","dist-tags":{"latest":"2.0.0"},
+ "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"},
+ "versions":{
+ "1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}},
+ "2.0.0":{"name":"testpkg","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-2.0.0.tgz"}}}}`
+ if _, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)); err != nil {
+ t.Fatalf("rewriteMetadataKeeping: %v", err)
+ }
+ assertWithheldRecorded(t, db, "npm", "testpkg", "2.0.0", recent)
+ assertNotRecorded(t, db, "npm", "testpkg", "1.0.0")
+}
+
+func TestCargoIndexRecordsWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &CargoHandler{proxy: proxy, proxyURL: "http://localhost:8080"}
+ line := func(version string, at time.Time) string {
+ return `{"name":"serde","vers":"` + version + `","cksum":"abcd","features":{},"yanked":false,"pubtime":"` + at.Format(time.RFC3339) + `"}` + "\n"
+ }
+ h.applyCooldownFiltering(httptest.NewRecorder(), []byte(line("1.0.0", old)+line("1.0.1", recent)))
+ assertWithheldRecorded(t, db, "cargo", "serde", "1.0.1", recent)
+ assertNotRecorded(t, db, "cargo", "serde", "1.0.0")
+}
+
+func TestComposerMetadataRecordsWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080"}
+ input := `{"packages":{"symfony/console":[
+ {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/5.0.0.zip","type":"zip"}},
+ {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/6.0.0.zip","type":"zip"}}]}}`
+ if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil {
+ t.Fatalf("rewriteMetadataKeeping: %v", err)
+ }
+ assertWithheldRecorded(t, db, "composer", "symfony/console", "6.0.0", recent)
+ assertNotRecorded(t, db, "composer", "symfony/console", "5.0.0")
+}
+
+func TestPubMetadataRecordsWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"}
+ input := `{"name":"flutter_bloc","latest":{"version":"2.0.0"},"versions":[
+ {"version":"1.0.0","published":"` + old.Format(time.RFC3339) + `","archive_url":"https://pub.dev/1.0.0.tar.gz"},
+ {"version":"2.0.0","published":"` + recent.Format(time.RFC3339) + `","archive_url":"https://pub.dev/2.0.0.tar.gz"}]}`
+ if _, err := h.rewriteMetadata("flutter_bloc", []byte(input)); err != nil {
+ t.Fatalf("rewriteMetadata: %v", err)
+ }
+ assertWithheldRecorded(t, db, "pub", "flutter_bloc", "2.0.0", recent)
+ assertNotRecorded(t, db, "pub", "flutter_bloc", "1.0.0")
+}
+
+func TestTimedDocumentsRecordWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ docs := timedDocuments{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent, "3.0.0": {}}}
+ for _, v := range []string{"1.0.0", "2.0.0", "3.0.0", "4.0.0"} {
+ docs.withheld(proxy, "hex", "testpkg", v)
+ }
+ assertWithheldRecorded(t, db, "hex", "testpkg", "2.0.0", recent)
+ for _, v := range []string{"1.0.0", "3.0.0", "4.0.0"} {
+ assertNotRecorded(t, db, "hex", "testpkg", v)
+ }
+}
+
+func TestGemIndexRecordsWithheldVersions(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ versionsJSON, _ := json.Marshal([]gemVersion{
+ {Number: "1.0.0", Platform: "ruby", CreatedAt: old.Format(time.RFC3339)},
+ {Number: "2.0.0", Platform: "ruby", CreatedAt: recent.Format(time.RFC3339)},
+ })
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.HasPrefix(r.URL.Path, "/info/"):
+ w.Header().Set("Content-Type", "text/plain")
+ _, _ = w.Write([]byte("---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n"))
+ case strings.HasPrefix(r.URL.Path, "/api/v1/versions/"):
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write(versionsJSON)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer upstream.Close()
+ h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"}
+
+ req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil)
+ req.SetPathValue("name", "testgem")
+ w := httptest.NewRecorder()
+ h.handleCompactIndex(w, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
+ }
+ assertWithheldRecorded(t, db, "gem", "testgem", "2.0.0", recent)
+ assertNotRecorded(t, db, "gem", "testgem", "1.0.0")
+}
+
+// Go stores the publish time of every version it reads, so what the filter
+// adds is the package row.
+func TestGoListRecordsWithheldVersions(t *testing.T) {
+ u := newGoCooldownUpstream(t, "v1.1.0")
+ h, _, proxy := goCooldownHandler(t, u)
+ if w := goGet(h, "/example.com/mod/@v/list"); w.Code != http.StatusOK {
+ t.Fatalf("status = %d", w.Code)
+ }
+ assertPackageRow(t, proxy.DB, "golang", "example.com/mod")
+}
+
+// A .info request for a version in cooldown is refused, and the version is
+// recorded with the publish time the refusal was based on.
+func TestGoRefusedInfoRecordsWithheldVersion(t *testing.T) {
+ u := newGoCooldownUpstream(t, "v1.1.0")
+ h, _, proxy := goCooldownHandler(t, u)
+ if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusForbidden {
+ t.Fatalf("status = %d, want 403", w.Code)
+ }
+ ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.1.0"))
+ if err != nil || ver == nil || !ver.PublishedAt.Valid {
+ t.Fatalf("version = %v, %v; want it stored with its publish time", ver, err)
+ }
+ assertPackageRow(t, proxy.DB, "golang", "example.com/mod")
+}
+
+func assertPackageRow(t *testing.T, db *database.DB, ecosystem, name string) {
+ t.Helper()
+ if pkg, err := db.GetPackageByEcosystemName(ecosystem, name); err != nil || pkg == nil {
+ t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err)
+ }
+}
+
+func TestMavenMetadataRecordsWithheldVersions(t *testing.T) {
+ u := newMavenCooldownUpstream(t)
+ h, proxy := newMavenCooldownHandler(t, u)
+ if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ for _, v := range []string{"2.0", "2.1"} {
+ assertWithheldRecorded(t, proxy.DB, "maven", "com.example:lib", v, u.times[v].UTC().Truncate(time.Second))
+ }
+}
+
+func TestMavenNamedRepositoryRecordsNoWithheldVersions(t *testing.T) {
+ u := newMavenCooldownUpstream(t)
+ proxy, db, _, _ := setupTestProxy(t)
+ proxy.HTTPClient = u.server.Client()
+ proxy.Cooldown = &cooldown.Config{Default: "3d"}
+ h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal").
+ WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes()
+ if w := getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ if pkg, err := db.GetPackageByEcosystemName("maven", "com.example:lib"); err != nil || pkg != nil {
+ t.Errorf("package = %v, %v; want no row from a named repository", pkg, err)
+ }
+}
+
+func TestPyPISimpleRecordsWithheldVersions(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ switch r.URL.Path {
+ case "/simple/ruff/":
+ body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` +
+ `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old.Format(time.RFC3339) + `"},` +
+ `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent.Format(time.RFC3339) + `"}]}`
+ return pypiHTTPResponse(r, pypiSimpleJSON, body), nil
+ case "/pypi/ruff/json":
+ body := `{"releases":{"1.0.0":[{"upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` +
+ `"2.0.0":[{"upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}`
+ return pypiHTTPResponse(r, "application/json", body), nil
+ default:
+ t.Fatalf("unexpected upstream path: %s", r.URL.Path)
+ return nil, nil
+ }
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+
+ req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil)
+ req.Header.Set("Accept", uvPyPIAccept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent)
+ assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0")
+}
+
+// The JSON API filters releases on its own path, so a client that reads it
+// rather than the simple index gets the held back version recorded too.
+func TestPyPIJSONRecordsWithheldVersions(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ if r.URL.Path != "/pypi/ruff/json" {
+ t.Fatalf("unexpected upstream path: %s", r.URL.Path)
+ }
+ body := `{"info":{"name":"ruff","version":"1.0.0"},"releases":{` +
+ `"1.0.0":[{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` +
+ `"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Add(-time.Minute).Format(time.RFC3339) + `"},` +
+ `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}`
+ return pypiHTTPResponse(r, "application/json", body), nil
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil))
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ if strings.Contains(w.Body.String(), `"2.0.0"`) {
+ t.Fatalf("2.0.0 was served although it is in cooldown")
+ }
+ // A release's time is its newest file's.
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent)
+ assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0")
+}
+
+// A release's time is its newest file's, and files can be added to a release
+// after it was first held back, so the newer time replaces the stored one.
+func TestPyPIWithheldReleaseKeepsItsNewestUploadTime(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ first, added := now.Add(-2*time.Hour), now.Add(-time.Hour)
+ files := []string{`{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + first.Format(time.RFC3339) + `"}`}
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ body := `{"info":{"name":"ruff","version":"2.0.0"},"releases":{"2.0.0":[` + strings.Join(files, ",") + `]}}`
+ return pypiHTTPResponse(r, "application/json", body), nil
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+ get := func() {
+ t.Helper()
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil))
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ }
+
+ get()
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first)
+ files = append(files, `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"`+added.Format(time.RFC3339)+`"}`)
+ get()
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added)
+}
+
+// The simple index and the version JSON also keep the newest upload time.
+func TestPyPIWithheldReleaseKeepsItsNewestUploadTimeOnEveryPath(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ first, added := now.Add(-2*time.Hour), now.Add(-time.Hour)
+ for name, path := range map[string]string{"simple": "/simple/ruff/", "version json": "/pypi/ruff/2.0.0/json"} {
+ t.Run(name, func(t *testing.T) {
+ times := []time.Time{first}
+ files := func(key string) string {
+ var out []string
+ for i, at := range times {
+ out = append(out, fmt.Sprintf(`{"filename":"ruff-2.0.0-%d.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0-%d.tar.gz",%q:%q}`, i, i, key, at.Format(time.RFC3339)))
+ }
+ return strings.Join(out, ",")
+ }
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ switch r.URL.Path {
+ case "/simple/ruff/":
+ return pypiHTTPResponse(r, pypiSimpleJSON, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[`+files("upload-time")+`]}`), nil
+ case "/pypi/ruff/json":
+ return pypiHTTPResponse(r, "application/json", `{"releases":{"2.0.0":[`+files("upload_time_iso_8601")+`]}}`), nil
+ default:
+ return pypiHTTPResponse(r, "application/json", `{"info":{"name":"ruff","version":"2.0.0"},"urls":[`+files("upload_time_iso_8601")+`]}`), nil
+ }
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+ get := func() {
+ t.Helper()
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ req.Header.Set("Accept", uvPyPIAccept)
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, req)
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ }
+ get()
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first)
+ times = append(times, added)
+ get()
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added)
+ })
+ }
+}
+
+// A download that read an older document than one already recorded leaves
+// the newer upload time, so cooldown doesn't end early.
+func TestPyPIDownloadKeepsTheNewerStoredUploadTime(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ older, newer := now.Add(-2*time.Hour), now.Add(-time.Hour)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ return pypiHTTPResponse(r, "application/json",
+ `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+ proxy.recordWithheldPyPI("ruff", "2.0.0", newer)
+
+ req := httptest.NewRequest(http.MethodGet, "/", nil)
+ if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown {
+ t.Fatalf("versionInCooldown = %v, %v; want true", inCooldown, err)
+ }
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", newer)
+}
+
+// A download that read an upload time out of cooldown is still refused when
+// a newer time, inside it, was stored meanwhile.
+func TestPyPIDownloadDecidesWithTheNewerStoredUploadTime(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ older, newer := now.Add(-30*24*time.Hour), now.Add(-time.Hour)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ return pypiHTTPResponse(r, "application/json",
+ `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+ proxy.recordWithheldPyPI("ruff", "2.0.0", newer)
+
+ req := httptest.NewRequest(http.MethodGet, "/", nil)
+ if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown {
+ t.Errorf("versionInCooldown = %v, %v; want true from the newer stored time", inCooldown, err)
+ }
+}
+
+// The version JSON names one release, in urls, and records it when it is
+// held back.
+func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) {
+ recent := time.Now().UTC().Truncate(time.Second).Add(-time.Hour)
+ h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) {
+ body := `{"info":{"name":"ruff","version":"2.0.0"},"urls":[` +
+ `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}`
+ return pypiHTTPResponse(r, "application/json", body), nil
+ })
+ proxy.Cooldown = &cooldown.Config{Default: "7d"}
+
+ w := httptest.NewRecorder()
+ h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/2.0.0/json", nil))
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ if strings.Contains(w.Body.String(), "ruff-2.0.0.tar.gz") {
+ t.Fatalf("the files of 2.0.0 were served although it is in cooldown")
+ }
+ assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent)
+}
+
+// The NuGet listing and the registration each find the held back version.
+func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) {
+ for _, path := range []string{
+ "/nuget/v3-flatcontainer/testpkg/index.json",
+ "/nuget" + nugetRegistrationPath + "testpkg/index.json",
+ } {
+ t.Run(path, func(t *testing.T) {
+ proxy, db, _, _ := setupTestProxy(t)
+ proxy.Cooldown = &cooldown.Config{Default: "14d"}
+ metadataRequests := 0
+ upstream := newNuGetCooldownUpstream(t, &metadataRequests)
+ defer upstream.Close()
+ h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL)
+ nugetGet(t, http.StripPrefix("/nuget", h.Routes()), path, http.StatusOK)
+
+ assertPackageRow(t, db, "nuget", "testpkg")
+ ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0"))
+ if err != nil || ver == nil || !ver.PublishedAt.Valid {
+ t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err)
+ }
+ if age := time.Since(ver.PublishedAt.Time); age < 47*time.Hour || age > 49*time.Hour {
+ t.Errorf("2.0.0 published %v ago, want two days", age)
+ }
+ assertNotRecorded(t, db, "nuget", "testpkg", "1.0.0")
+ })
+ }
+}
+
+// Without a name the PURLs would name no package, such as pkg:cargo/.
+func TestRecordWithheldSkipsPackagesWithoutAName(t *testing.T) {
+ proxy, db, _, recent := withheldTestProxy(t)
+ proxy.recordWithheld("cargo", "", "1.0.0", recent)
+ if pkg, err := db.GetPackageByEcosystemName("cargo", ""); err != nil || pkg != nil {
+ t.Errorf("package = %v, %v; want nothing recorded without a name", pkg, err)
+ }
+}
+
+// A routed package's times come from its route and are never stored under
+// the default registry's PURLs, so the filter doesn't record them either.
+func TestNPMMetadataOfRoutedPackageRecordsNothing(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080",
+ routes: newPackageRoutes(map[string]string{"@corp/*": "https://npm.corp.example"})}
+ input := `{"name":"@corp/lib","dist-tags":{"latest":"2.0.0"},
+ "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"},
+ "versions":{
+ "1.0.0":{"name":"@corp/lib","version":"1.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-1.0.0.tgz"}},
+ "2.0.0":{"name":"@corp/lib","version":"2.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-2.0.0.tgz"}}}}`
+ if _, _, err := h.rewriteMetadataKeeping("@corp/lib", []byte(input)); err != nil {
+ t.Fatalf("rewriteMetadataKeeping: %v", err)
+ }
+ assertNotRecorded(t, db, "npm", "@corp/lib", "2.0.0")
+}
+
+func TestComposerMetadataOfRoutedPackageRecordsNothing(t *testing.T) {
+ proxy, db, old, recent := withheldTestProxy(t)
+ h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080",
+ routes: newPackageRoutes(map[string]string{"corp/*": "https://composer.corp.example"})}
+ input := `{"packages":{"corp/lib":[
+ {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/5.0.0.zip","type":"zip"}},
+ {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/6.0.0.zip","type":"zip"}}]}}`
+ if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil {
+ t.Fatalf("rewriteMetadataKeeping: %v", err)
+ }
+ assertNotRecorded(t, db, "composer", "corp/lib", "6.0.0")
+}
+
+// Go keeps publish times under the escaped module path, which differs from
+// the module path downloads are stored under when it has capitals. Recording
+// such a module would give one release two version rows, so it is skipped.
+func TestGoListOfModuleWithCapitalsRecordsNoSecondRow(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/example.com/!mod/@v/list":
+ _, _ = w.Write([]byte("v1.1.0\n"))
+ case "/example.com/!mod/@v/v1.1.0.info":
+ _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.1.0", Time: now.Add(-time.Hour)})
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(upstream.Close)
+ proxy, db, _, _ := setupTestProxy(t)
+ proxy.Cooldown = &cooldown.Config{Default: "3d"}
+ h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes()
+ if w := goGet(h, "/example.com/!mod/@v/list"); w.Code != http.StatusOK {
+ t.Fatalf("status = %d: %s", w.Code, w.Body)
+ }
+ if ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/!mod", "v1.1.0")); err != nil || ver == nil {
+ t.Fatalf("time under the escaped path = %v, %v; want it stored", ver, err)
+ }
+ assertNotRecorded(t, db, "golang", "example.com/Mod", "v1.1.0")
+}
+
+// A registration leaf carries no version of its own; it is the one its path
+// names.
+func TestNuGetRegistrationLeafRecordsWithheldVersion(t *testing.T) {
+ proxy, db, _, _ := setupTestProxy(t)
+ proxy.Cooldown = &cooldown.Config{Default: "14d"}
+ metadataRequests := 0
+ upstream := newNuGetCooldownUpstream(t, &metadataRequests)
+ defer upstream.Close()
+ h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL)
+ nugetGet(t, http.StripPrefix("/nuget", h.Routes()), "/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound)
+
+ ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0"))
+ if err != nil || ver == nil || !ver.PublishedAt.Valid {
+ t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err)
+ }
+}
+
+func TestNuGetRegistrationLeafVersion(t *testing.T) {
+ for path, want := range map[string]string{
+ nugetRegistrationPath + "testpkg/2.0.0.json": "2.0.0",
+ nugetRegistrationPath + "testpkg/index.json": "",
+ nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": "",
+ nugetRegistrationPath + "testpkg/2.0.0": "",
+ "/other/testpkg/2.0.0.json": "",
+ } {
+ if got := nugetRegistrationLeafVersion(path); got != want {
+ t.Errorf("nugetRegistrationLeafVersion(%q) = %q, want %q", path, got, want)
+ }
+ }
+}
diff --git a/internal/handler/gem.go b/internal/handler/gem.go
index fa7c24ba..25983cdc 100644
--- a/internal/handler/gem.go
+++ b/internal/handler/gem.go
@@ -176,7 +176,6 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request)
return
}
- packagePURL := canonicalPackagePURL("gem", name)
m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(),
func([]byte) ([]byte, []string, error) {
body, err := h.proxy.decodedMetadataBody(docs.doc)
@@ -184,7 +183,7 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request)
return nil, nil, err
}
out, kept := h.filterIndex(body, name, func(version string) bool {
- return docs.withheld(h.proxy, "gem", packagePURL, version)
+ return docs.withheld(h.proxy, "gem", name, version)
})
return out, kept, nil
})
diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go
index efcc1e19..e788e688 100644
--- a/internal/handler/go_cooldown.go
+++ b/internal/handler/go_cooldown.go
@@ -168,6 +168,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s
}
keep := make([]bool, len(versions))
+ times := make([]time.Time, len(versions))
g, gctx := errgroup.WithContext(ctx)
g.SetLimit(goInfoFetchLimit)
for i, v := range versions {
@@ -177,6 +178,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s
return fmt.Errorf("reading time of %s: %w", v, err)
}
keep[i] = h.allowedAt(module, published)
+ times[i] = published
return nil
})
}
@@ -190,11 +192,23 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s
allowed = append(allowed, v)
} else {
h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v)
+ h.recordWithheld(module, v, times[i])
}
}
return allowed, nil
}
+// recordWithheld records a version the list filter held back. Publish times
+// are stored under the escaped module path and downloads under the module
+// path, so a module whose path has capitals, and the two differ, is skipped
+// rather than given a second row for the same release.
+func (h *GoHandler) recordWithheld(module, version string, published time.Time) {
+ if escaped, err := gomodule.EscapePath(module); err != nil || escaped != module {
+ return
+ }
+ h.proxy.recordWithheld("golang", module, version, published)
+}
+
// serveCooldownList serves @v/list without versions in cooldown.
func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) {
allowed, err := h.allowedVersions(r.Context(), escapedModule, module)
@@ -220,6 +234,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es
var info goInfo
if err := json.Unmarshal(body, &info); err == nil && info.Version != "" {
if !h.allowedAt(module, info.Time) {
+ h.recordWithheld(module, info.Version, info.Time)
h.refuseCooldown(w, module, info.Version)
return
}
diff --git a/internal/handler/hex.go b/internal/handler/hex.go
index d394c4df..17899f10 100644
--- a/internal/handler/hex.go
+++ b/internal/handler/hex.go
@@ -130,7 +130,6 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) {
return
}
- packagePURL := canonicalPackagePURL("hex", name)
m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(),
func([]byte) ([]byte, []string, error) {
body, err := h.proxy.decodedMetadataBody(docs.doc)
@@ -144,7 +143,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) {
filtered := make(map[string]bool)
var kept []string
for _, version := range versions {
- if docs.withheld(h.proxy, "hex", packagePURL, version) {
+ if docs.withheld(h.proxy, "hex", name, version) {
filtered[version] = true
h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version)
continue
diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go
index 6155e0d7..beccb703 100644
--- a/internal/handler/maven_cooldown.go
+++ b/internal/handler/maven_cooldown.go
@@ -330,6 +330,11 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a
}
h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v)
removed[v] = true
+ // A named repository's times are its own, under PURLs that name it,
+ // and the package page shows the default repository's.
+ if source.name == "" {
+ h.proxy.recordWithheld("maven", group+":"+artifact, v, published)
+ }
}
if len(removed) == 0 {
return body, nil
diff --git a/internal/handler/npm.go b/internal/handler/npm.go
index 3cf3244d..95308b78 100644
--- a/internal/handler/npm.go
+++ b/internal/handler/npm.go
@@ -721,6 +721,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l
// cooldown is evaluated at, and the pushed versions to list. The callbacks
// log what the rewrite did.
func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options {
+ _, routed := h.routes.match(packageName)
return npm.Options{
ProxyURL: h.proxyURL,
Cooldown: h.proxy.cooldown(),
@@ -736,6 +737,11 @@ func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry
OnCooldownFilter: func(version, publishedAt string) {
h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version,
"published", publishedAt)
+ // A routed package's times are the route's, and aren't stored
+ // under the default registry's PURLs (see versionInCooldown).
+ if at, err := time.Parse(time.RFC3339, publishedAt); err == nil && !routed {
+ h.proxy.recordWithheld("npm", packageName, version, at)
+ }
},
}
}
diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go
index 714f2192..1e6615a5 100644
--- a/internal/handler/nuget_cooldown.go
+++ b/internal/handler/nuget_cooldown.go
@@ -393,21 +393,27 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str
return false, err
}
_, published := leaf.release()
- return h.nugetPublishedAllowed(id, published), nil
+ return h.nugetPublishedAllowed(id, version, published), nil
}
// Missing/invalid timestamps retain the existing permissive behavior, but
-// fetch and JSON errors must not bypass the policy.
-func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool {
+// fetch and JSON errors must not bypass the policy. A version held back is
+// recorded under the lowercase ID and version downloads are stored under.
+func (h *NuGetHandler) nugetPublishedAllowed(id, version, published string) bool {
when, err := time.Parse(time.RFC3339, published)
if err != nil {
return true
}
- return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when)
+ name := strings.ToLower(id)
+ if h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", name), when) {
+ return true
+ }
+ h.proxy.recordWithheld("nuget", name, nugetVersionKey(version), when)
+ return false
}
func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) {
- if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) {
+ if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, version, published) {
blocked[nugetVersionKey(version)] = true
}
for i := range document.Items {
@@ -436,7 +442,11 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
return
}
if applyCooldown && document.Items == nil {
- if _, published := document.release(); !h.nugetPublishedAllowed(id, published) {
+ version, published := document.release()
+ if version == "" {
+ version = nugetRegistrationLeafVersion(r.URL.Path)
+ }
+ if !h.nugetPublishedAllowed(id, version, published) {
JSONError(w, http.StatusNotFound, "version not found")
return
}
@@ -469,6 +479,25 @@ func nugetRegistrationID(path string) string {
return ""
}
+// nugetRegistrationLeafVersion returns the version a registration leaf's
+// path names ({prefix}{id}/{version}.json), or "" for any other path. A leaf
+// as nuget.org serves it carries no version of its own.
+func nugetRegistrationLeafVersion(path string) string {
+ for _, prefix := range nugetRegistrationPrefixes {
+ rest, ok := strings.CutPrefix(path, prefix)
+ if !ok {
+ continue
+ }
+ _, leaf, _ := strings.Cut(rest, "/")
+ version, isJSON := strings.CutSuffix(leaf, ".json")
+ if !isJSON || version == "" || version == "index" || strings.Contains(version, "/") {
+ return ""
+ }
+ return version
+ }
+ return ""
+}
+
// expandNuGetPageTimes gives each page of a registration index that only
// links to its leaves the leaves of the page it links to. With bodies not nil
// the linked pages themselves are kept in it, by link. Only index pages are
diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go
index 01da55de..e32273ff 100644
--- a/internal/handler/nuget_registration.go
+++ b/internal/handler/nuget_registration.go
@@ -389,7 +389,11 @@ func (rw *nugetRegistrationWriter) node(dec *jsontext.Decoder, depth int) (bool,
if n.container {
return n.kept > 0, release.version, nil
}
- return rw.h.nugetPublishedAllowed(rw.id, release.published), release.version, nil
+ var version string
+ if len(release.version) >= 2 && release.version[0] == '"' {
+ version = string(rw.unquote(release.version))
+ }
+ return rw.h.nugetPublishedAllowed(rw.id, version, release.published), release.version, nil
}
// linksPage reports whether a node at depth is a page of an index whose
diff --git a/internal/handler/pub.go b/internal/handler/pub.go
index 5a6d426c..1fe8f040 100644
--- a/internal/handler/pub.go
+++ b/internal/handler/pub.go
@@ -288,6 +288,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap
if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) {
h.proxy.Logger.Info("cooldown: filtering pub version",
"package", name, "version", version)
+ h.proxy.recordWithheld("pub", name, version, publishedAt)
return true
}
diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go
index c94e1515..393e02be 100644
--- a/internal/handler/pypi.go
+++ b/internal/handler/pypi.go
@@ -4,7 +4,6 @@ import (
"bytes"
"crypto/sha256"
"encoding/hex"
- "encoding/json"
"errors"
"fmt"
"io"
@@ -12,12 +11,12 @@ import (
"mime"
"net/http"
"net/url"
- "regexp"
"slices"
"strconv"
"strings"
"time"
+ "github.com/git-pkgs/proxy/internal/metadata/pypi"
"github.com/git-pkgs/proxy/internal/retention"
)
@@ -30,17 +29,11 @@ const (
pypiSimpleLatestHTML = "application/vnd.pypi.simple.latest+html"
pypiLegacyHTML = "text/html"
pypiExactSpecificity = 2
- minWheelParts = 5 // name + version + python + abi + platform
- minSubmatchParts = 2 // full match + first capture group
minPyPIPathParts = 3 // hash_prefix + hash + filename
- minEggParts = 3 // name + version + python tag
-
- // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip
- // appends to a distribution URL when the index advertises core metadata.
- // A sidecar resolves to the same name and version as the distribution it
- // describes, so it is cached alongside it; consumers that expect an openable
- // archive must skip these.
- PyPIMetadataSuffix = ".metadata"
+
+ // PyPIMetadataSuffix is the PEP 658 core-metadata sidecar suffix; see
+ // pypi.MetadataSuffix.
+ PyPIMetadataSuffix = pypi.MetadataSuffix
)
// PyPIHandler handles PyPI registry protocol requests.
@@ -326,6 +319,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s
for version, publishedAt := range times {
if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) {
filtered[version] = true
+ h.proxy.recordWithheldPyPI(name, version, publishedAt)
}
}
@@ -352,159 +346,50 @@ func (h *PyPIHandler) fetchReleaseTimes(r *http.Request, name string) (map[strin
return h.parseReleaseTimes(body)
}
-// pypiReleaseFiles is the part of a JSON API document the release times are
-// read from. Decoding into it skips every other field, where a generic
-// map would hold the whole document, several times its size, in memory.
-type pypiReleaseFiles struct {
- Releases map[string][]struct {
- UploadTime string `json:"upload_time_iso_8601"`
- } `json:"releases"`
-}
-
// parseReleaseTimes returns each release's newest upload time from a JSON
// API document, zero for a release without one.
func (h *PyPIHandler) parseReleaseTimes(body []byte) (map[string]time.Time, error) {
- var metadata pypiReleaseFiles
- // A value of another type than expected is skipped and the rest is
- // still decoded, so a file entry without a usable time doesn't count.
- var mismatch *json.UnmarshalTypeError
- if err := json.Unmarshal(body, &metadata); err != nil && !errors.As(err, &mismatch) {
- return nil, fmt.Errorf("parsing pypi json metadata: %w", err)
- }
- if metadata.Releases == nil {
- return nil, errors.New("pypi json metadata has no releases")
- }
-
- times := make(map[string]time.Time, len(metadata.Releases))
- for version, files := range metadata.Releases {
- var newest time.Time
- for _, file := range files {
- if t, err := time.Parse(time.RFC3339, file.UploadTime); err == nil && t.After(newest) {
- newest = t
- }
- }
- times[version] = newest
- }
- return times, nil
+ return pypi.ReleaseTimes(body)
}
-var (
- // simpleLinkRe matches a whole anchor tag: filename.
- simpleLinkRe = regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`)
- // simpleLinkTextRe captures an anchor's text.
- simpleLinkTextRe = regexp.MustCompile(`>([^<]+)`)
-)
+// urlOptions are the URLs a simple index rewrite needs; the simple index is
+// filtered by the versions the caller passes.
+func (h *PyPIHandler) urlOptions() pypi.Options {
+ return pypi.Options{ProxyURL: h.proxyURL, DownloadURL: h.downloadURL}
+}
-// rewriteSimpleHTML rewrites package URLs in simple API HTML to point at this proxy.
-// If filteredVersions is non-nil, links for those versions are removed entirely.
-func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string]bool) []byte {
- // If cooldown filtering is active, remove entire tags for filtered versions
- if len(filteredVersions) > 0 {
- body = simpleLinkRe.ReplaceAllFunc(body, func(match []byte) []byte {
- // Extract filename from between tags
- innerMatch := simpleLinkTextRe.FindSubmatch(match)
- if len(innerMatch) < minSubmatchParts {
- return match
- }
- filename := string(innerMatch[1])
- _, version := h.parseFilename(strings.TrimSpace(filename))
- if version != "" && filteredVersions[version] {
- return nil
- }
- return match
- })
+// metadataOptions add the policy the JSON API rewrite applies itself. The
+// callbacks log and record what it holds back.
+func (h *PyPIHandler) metadataOptions() pypi.Options {
+ return pypi.Options{
+ ProxyURL: h.proxyURL,
+ DownloadURL: h.downloadURL,
+ Cooldown: h.proxy.cooldown(),
+ EvaluatedAt: time.Now(),
+ Denied: func(name, version string) bool {
+ return h.proxy.versionDenied("pypi", name, version)
+ },
+ OnFiltered: func(name, version string) {
+ h.proxy.Logger.Info("policy: filtering pypi version", "package", name, "version", version)
+ },
+ OnWithheld: h.proxy.recordWithheldPyPI,
}
+}
- return h.rewriteDownloadHrefs(body)
+func (h *PyPIHandler) rewriteSimpleHTML(body []byte, filteredVersions map[string]bool) []byte {
+ return pypi.RewriteSimpleHTML(body, filteredVersions, h.urlOptions())
}
-// rewriteDownloadHrefs points href attributes for files on the configured
-// download host at the proxy. It makes the same replacements as the regular
-// expression `href="(/packages/[^"]+)"` would: an href
-// whose URL starts with the download host's /packages/ and has at least one
-// more character before the closing quote. Finding the literal prefix is many
-// times faster than running the regular expression over every link.
func (h *PyPIHandler) rewriteDownloadHrefs(body []byte) []byte {
- from := []byte(`href="` + h.downloadURL + `/packages/`)
- // The download route is /pypi/packages/{path...} and keeps the path
- // from the download host, /packages/ included.
- to := []byte(`href="` + h.proxyURL + `/pypi/packages/packages/`)
-
- var out []byte
- done := 0 // body[:done] is in out already
- for search := 0; ; {
- i := bytes.Index(body[search:], from)
- if i < 0 {
- break
- }
- start := search + i
- rest := start + len(from)
- end := bytes.IndexByte(body[rest:], '"')
- if end <= 0 {
- // No closing quote, or nothing before it: not a match. The
- // regular expression would go on looking from the next byte.
- search = start + 1
- continue
- }
- if out == nil {
- out = make([]byte, 0, len(body)+len(body)/8)
- }
- out = append(out, body[done:start]...)
- out = append(out, to...)
- done = rest
- search = rest + end + 1
- }
- if out == nil {
- return body
- }
- return append(out, body[done:]...)
+ return pypi.RewriteDownloadHrefs(body, h.urlOptions())
}
-func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) {
- var metadata map[string]any
- if err := json.Unmarshal(body, &metadata); err != nil {
- return nil, err
- }
-
- files, ok := metadata["files"].([]any)
- if !ok {
- return nil, errors.New("pypi simple json response has no files array")
- }
-
- rewrittenFiles := make([]any, 0, len(files))
- for _, file := range files {
- entry, ok := file.(map[string]any)
- if !ok {
- rewrittenFiles = append(rewrittenFiles, file)
- continue
- }
-
- if filename, ok := entry["filename"].(string); ok {
- _, version := h.parseFilename(filename)
- if version != "" && filteredVersions[version] {
- continue
- }
- }
-
- h.rewriteURLEntry(entry)
- rewrittenFiles = append(rewrittenFiles, entry)
- }
-
- metadata["files"] = rewrittenFiles
+func (h *PyPIHandler) filterSimpleHTMLLinks(body []byte, versions map[string]bool) []byte {
+ return pypi.FilterSimpleHTMLLinks(body, versions)
+}
- // The versions list (PEP 700) loses the filtered versions too, whose
- // files were all left out above.
- if versions, ok := metadata["versions"].([]any); ok && len(filteredVersions) > 0 {
- keptVersions := make([]any, 0, len(versions))
- for _, v := range versions {
- if version, ok := v.(string); ok && filteredVersions[version] {
- continue
- }
- keptVersions = append(keptVersions, v)
- }
- metadata["versions"] = keptVersions
- }
- return json.Marshal(metadata)
+func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string]bool) ([]byte, error) {
+ return pypi.RewriteSimpleJSON(body, filteredVersions, h.urlOptions())
}
// handleJSON serves the JSON API package metadata.
@@ -584,71 +469,10 @@ func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request
serveRewrittenMetadata(w, r, rewritten)
}
-// rewriteJSONMetadata rewrites download URLs in PyPI JSON metadata.
-// If cooldown is enabled, versions published too recently are filtered out.
-// It also returns what it kept: the releases, and the current version's files
-// unless they were dropped.
+// rewriteJSONMetadata rewrites download URLs in PyPI JSON metadata, leaving
+// out releases the denylist names or cooldown holds back.
func (h *PyPIHandler) rewriteJSONMetadata(body []byte) ([]byte, []string, error) {
- var metadata map[string]any
- if err := json.Unmarshal(body, &metadata); err != nil {
- return nil, nil, err
- }
-
- packageName, _ := extractPyPIName(metadata)
- packagePURL := ""
- if packageName != "" {
- packagePURL = canonicalPackagePURL("pypi", packageName)
- }
-
- h.filterAndRewriteReleases(metadata, packageName, packagePURL)
- h.filterAndRewriteURLs(metadata, packagePURL)
-
- var kept []string
- if releases, ok := metadata["releases"].(map[string]any); ok {
- kept = slices.Collect(maps.Keys(releases))
- }
- if urls, ok := metadata["urls"].([]any); ok && len(urls) > 0 {
- // Not a valid release key, so it can't be confused with one.
- kept = append(kept, "\x00urls")
- }
-
- out, err := json.Marshal(metadata)
- return out, kept, err
-}
-
-// filterAndRewriteReleases applies cooldown filtering and URL rewriting to the
-// releases map in PyPI metadata.
-func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageName, packagePURL string) {
- releases, ok := metadata["releases"].(map[string]any)
- if !ok {
- return
- }
-
- for version, files := range releases {
- if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) {
- h.proxy.Logger.Info("policy: filtering pypi version",
- "package", packageName, "version", version)
- delete(releases, version)
- continue
- }
-
- h.rewriteFileEntries(files)
- }
-}
-
-// shouldFilterRelease returns true if a release should be excluded due to cooldown.
-func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool {
- if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() || packagePURL == "" {
- return false
- }
-
- filesArr, ok := files.([]any)
- if !ok {
- return false
- }
-
- publishedAt := h.newestUploadTime(filesArr)
- return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt)
+ return pypi.RewriteJSON(body, h.metadataOptions())
}
// versionInCooldown reports whether a version is still inside the cooldown
@@ -658,7 +482,8 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool {
//
// A release whose upload time cannot be determined is allowed through, matching
// how fetchFilteredVersions treats it. Failing to read the times is an error.
-// The time read is stored for storedCooldown.
+// The time read is stored for storedCooldown, and a newer one stored
+// meanwhile decides.
func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) {
if !h.proxy.cooldownApplies("pypi", name) {
return false, nil
@@ -672,24 +497,32 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (
if publishedAt.IsZero() {
return false, nil
}
- h.storeUploadTime(name, version, publishedAt)
+ publishedAt = h.storeUploadTime(name, version, publishedAt)
return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil
}
// storeUploadTime records a release's newest upload time under the version
-// PURL the file is cached under, so storedCooldown finds it.
-func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) {
+// PURL the file is cached under, so storedCooldown finds it. The time only
+// moves forward: a request that read an older document leaves a newer one,
+// and gets that newer time back.
+func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) time.Time {
if h.proxy.DB == nil {
- return
+ return publishedAt
}
packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version)
if err != nil {
- return
+ return publishedAt
}
- if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil {
+ if err := h.proxy.DB.AdvanceVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil {
h.proxy.Logger.Warn("cooldown: could not store pypi upload time",
"package", name, "version", version, "error", err)
+ return publishedAt
+ }
+ if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil &&
+ ver.PublishedAt.Valid && ver.PublishedAt.Time.After(publishedAt) {
+ return ver.PublishedAt.Time
}
+ return publishedAt
}
// storedCooldown answers the cooldown check for a download from the upload
@@ -719,89 +552,6 @@ func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename st
return false, err == nil && cached
}
-// rewriteFileEntries rewrites URLs in a list of file entries.
-func (h *PyPIHandler) rewriteFileEntries(files any) {
- filesArr, ok := files.([]any)
- if !ok {
- return
- }
-
- for _, f := range filesArr {
- if fmap, ok := f.(map[string]any); ok {
- h.rewriteURLEntry(fmap)
- }
- }
-}
-
-// filterAndRewriteURLs applies cooldown filtering and URL rewriting to the
-// urls array (current version files) in PyPI metadata.
-func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL string) {
- urls, ok := metadata["urls"].([]any)
- if !ok {
- return
- }
-
- info, _ := metadata["info"].(map[string]any)
- name, _ := info["name"].(string)
- version, _ := info["version"].(string)
- if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) {
- metadata["urls"] = []any{}
- }
-
- if urls, ok := metadata["urls"].([]any); ok {
- for _, u := range urls {
- if umap, ok := u.(map[string]any); ok {
- h.rewriteURLEntry(umap)
- }
- }
- }
-}
-
-// extractPyPIName extracts the package name from PyPI JSON metadata.
-func extractPyPIName(metadata map[string]any) (string, bool) {
- info, ok := metadata["info"].(map[string]any)
- if !ok {
- return "", false
- }
- name, ok := info["name"].(string)
- return name, ok
-}
-
-// newestUploadTime returns the most recent upload_time_iso_8601 from a list of file entries.
-func (h *PyPIHandler) newestUploadTime(files []any) time.Time {
- var newest time.Time
- for _, f := range files {
- fmap, ok := f.(map[string]any)
- if !ok {
- continue
- }
- ts, ok := fmap["upload_time_iso_8601"].(string)
- if !ok {
- continue
- }
- t, err := time.Parse(time.RFC3339, ts)
- if err != nil {
- continue
- }
- if t.After(newest) {
- newest = t
- }
- }
- return newest
-}
-
-// rewriteURLEntry rewrites a single URL entry in PyPI metadata.
-func (h *PyPIHandler) rewriteURLEntry(entry map[string]any) {
- urlStr, ok := entry["url"].(string)
- if !ok {
- return
- }
-
- if strings.HasPrefix(urlStr, h.downloadURL+"/packages/") {
- entry["url"] = h.proxyURL + "/pypi/packages" + strings.TrimPrefix(urlStr, h.downloadURL)
- }
-}
-
// handleDownload serves a package file, fetching and caching from upstream if needed.
func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
path := r.PathValue("path")
@@ -865,171 +615,8 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
// archiveExtensions are sdist formats of the form {name}-{version}{ext}. They
// carry no trailing tags, but legacy sdist names may contain hyphens.
-var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"}
-
-// windowsInstallerExtensions are the legacy distutils bdist_wininst and
-// bdist_msi formats, which share a filename layout.
-var windowsInstallerExtensions = []string{".exe", ".msi"}
-
func (h *PyPIHandler) parseFilename(filename string) (name, version string) {
- return PyPIFilenameIdentity(filename)
-}
-
-// PyPIFilenameIdentity extracts the package name and version a PyPI file is
-// cached under from its filename, so proxy mirror stores files where the
-// download handler looks for them. Handles wheels, sdists and legacy bdist
-// formats:
-// - requests-2.31.0-py3-none-any.whl
-// - requests-2.31.0.tar.gz
-// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg
-// - numpy-1.8.0.win32-py2.7.exe
-func PyPIFilenameIdentity(filename string) (name, version string) {
- // PEP 658/714 core-metadata sidecars are the distribution filename plus
- // ".metadata"; they describe the same name and version. Without this, pip's
- // metadata-only fetches fall back to a hash-derived package identifier.
- filename = strings.TrimSuffix(filename, PyPIMetadataSuffix)
-
- switch {
- case strings.HasSuffix(filename, ".whl"):
- return parseWheelFilename(strings.TrimSuffix(filename, ".whl"))
- case strings.HasSuffix(filename, ".egg"):
- return parseEggFilename(strings.TrimSuffix(filename, ".egg"))
- }
-
- for _, ext := range windowsInstallerExtensions {
- if strings.HasSuffix(filename, ext) {
- return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext))
- }
- }
-
- for _, ext := range archiveExtensions {
- if strings.HasSuffix(filename, ext) {
- return splitNameVersion(strings.TrimSuffix(filename, ext))
- }
- }
-
- return "", ""
-}
-
-// parseWheelFilename parses the PEP 427 layout
-// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the
-// filename without its ".whl" suffix. The spec escapes every hyphen in the name
-// and version to '_', so the first two fields are authoritative even when the
-// optional build tag is present.
-func parseWheelFilename(base string) (name, version string) {
- parts := strings.Split(base, "-")
- if len(parts) < minWheelParts {
- return "", ""
- }
-
- return parts[0], parts[1]
-}
-
-// parseEggFilename parses the setuptools bdist_egg layout
-// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its
-// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but
-// eggs built by other tooling do not always, so the version is located relative
-// to the interpreter field rather than assumed to be the second field.
-func parseEggFilename(base string) (name, version string) {
- parts := strings.Split(base, "-")
- // Scan from the end: the trailing platform fields never look like an
- // interpreter tag, so the last match is the real one even when the package
- // name itself carries a "py{N}" component. Stop before index 1, since a tag
- // any earlier would leave no room for both a name and a version.
- for i := len(parts) - 1; i >= minEggParts-1; i-- {
- if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) {
- continue
- }
-
- return strings.Join(parts[:i-1], "-"), parts[i-1]
- }
-
- // No interpreter field: {name}-{version}.
- return splitNameVersion(base)
-}
-
-// parseWindowsInstallerFilename parses the distutils bdist_wininst and
-// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the
-// filename without its ".exe" or ".msi" suffix. The platform is joined to the
-// version with a '.' rather than a '-' and may itself contain a hyphen
-// ("win-amd64"), so both trailing fields are stripped before the name and
-// version are split apart.
-func parseWindowsInstallerFilename(base string) (name, version string) {
- if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) {
- base = base[:i]
- }
-
- // The platform is the final '.'-separated field. Requiring it to start with
- // a non-digit keeps a dotted version from being truncated when a filename
- // carries no platform tag.
- i := strings.LastIndex(base, ".")
- if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) {
- return "", ""
- }
-
- return splitFullname(base[:i])
-}
-
-// splitFullname splits the distutils fullname {name}-{version} that precedes a
-// Windows installer's platform field. Unlike an sdist, a wininst fullname may
-// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to
-// neither the name nor the version, so the first purely numeric field wins and
-// anything after it is discarded.
-func splitFullname(fullname string) (name, version string) {
- parts := strings.Split(fullname, "-")
- for i := 1; i < len(parts); i++ {
- if isDottedNumber(parts[i]) {
- return strings.Join(parts[:i], "-"), parts[i]
- }
- }
-
- // No purely numeric field, e.g. a prerelease version like "1.0b1".
- return splitNameVersion(fullname)
-}
-
-// splitNameVersion splits a {name}-{version} pair at the last hyphen that
-// starts a version, leaving hyphens inside the name intact.
-func splitNameVersion(base string) (name, version string) {
- for i := len(base) - 1; i >= 0; i-- {
- if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) {
- return base[:i], base[i+1:]
- }
- }
-
- return "", ""
-}
-
-// isEggPythonTag reports whether field is the py{X.Y} interpreter field that
-// setuptools places directly after the version in an egg filename.
-func isEggPythonTag(field string) bool {
- const prefix = "py"
-
- return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)])
-}
-
-// isVersionField reports whether field can be a version, i.e. it is non-empty
-// and starts with a digit as every PEP 440 release segment does.
-func isVersionField(field string) bool {
- return field != "" && isVersionStart(field[0])
-}
-
-// isDottedNumber reports whether s is a dotted numeric version such as "2.7".
-func isDottedNumber(s string) bool {
- if s == "" || !isVersionStart(s[0]) {
- return false
- }
-
- for i := range len(s) {
- if !isVersionStart(s[i]) && s[i] != '.' {
- return false
- }
- }
-
- return true
-}
-
-func isVersionStart(c byte) bool {
- return c >= '0' && c <= '9'
+ return pypi.FilenameIdentity(filename)
}
func hashPath(path string) string {
diff --git a/internal/handler/pypi_denylist.go b/internal/handler/pypi_denylist.go
deleted file mode 100644
index f39c0d8e..00000000
--- a/internal/handler/pypi_denylist.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package handler
-
-import (
- "bytes"
- "net/url"
- "path"
-
- "golang.org/x/net/html"
-)
-
-// filterSimpleHTMLLinks reads hrefs rather than display text. Tokenization
-// handles single quotes, entity escaping, and nested link text without
-// reserializing the rest of the index.
-func (h *PyPIHandler) filterSimpleHTMLLinks(body []byte, versions map[string]bool) []byte {
- if len(versions) == 0 {
- return body
- }
- var result bytes.Buffer
- z := html.NewTokenizer(bytes.NewReader(body))
- skip := false
- for {
- tokenType := z.Next()
- if tokenType == html.ErrorToken {
- return result.Bytes()
- }
- raw := append([]byte(nil), z.Raw()...)
- if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
- token := z.Token()
- if token.Data == "a" {
- skip = h.deniedSimpleLink(token, versions)
- }
- } else if tokenType == html.EndTagToken && z.Token().Data == "a" {
- if skip {
- skip = false
- continue
- }
- }
- if !skip {
- result.Write(raw)
- }
- }
-}
-
-func (h *PyPIHandler) deniedSimpleLink(token html.Token, versions map[string]bool) bool {
- for _, attr := range token.Attr {
- if attr.Key != "href" {
- continue
- }
- u, err := url.Parse(attr.Val)
- if err != nil {
- continue
- }
- _, version := h.parseFilename(path.Base(u.Path))
- if versions[version] {
- return true
- }
- }
- return false
-}
diff --git a/internal/metadata/pypi/filename.go b/internal/metadata/pypi/filename.go
new file mode 100644
index 00000000..277693cf
--- /dev/null
+++ b/internal/metadata/pypi/filename.go
@@ -0,0 +1,178 @@
+package pypi
+
+import "strings"
+
+const (
+ minWheelParts = 5 // name + version + python + abi + platform
+ minEggParts = 3 // name + version + python tag
+
+ // MetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip
+ // appends to a distribution URL when the index advertises core metadata.
+ // A sidecar resolves to the same name and version as the distribution it
+ // describes, so it is cached alongside it; consumers that expect an openable
+ // archive must skip these.
+ MetadataSuffix = ".metadata"
+)
+
+var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"}
+
+// windowsInstallerExtensions are the legacy distutils bdist_wininst and
+// bdist_msi formats, which share a filename layout.
+var windowsInstallerExtensions = []string{".exe", ".msi"}
+
+// FilenameIdentity extracts the package name and version a PyPI file is
+// cached under from its filename, so proxy mirror stores files where the
+// download handler looks for them. Handles wheels, sdists and legacy bdist
+// formats:
+// - requests-2.31.0-py3-none-any.whl
+// - requests-2.31.0.tar.gz
+// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg
+// - numpy-1.8.0.win32-py2.7.exe
+func FilenameIdentity(filename string) (name, version string) {
+ // PEP 658/714 core-metadata sidecars are the distribution filename plus
+ // ".metadata"; they describe the same name and version. Without this, pip's
+ // metadata-only fetches fall back to a hash-derived package identifier.
+ filename = strings.TrimSuffix(filename, MetadataSuffix)
+
+ switch {
+ case strings.HasSuffix(filename, ".whl"):
+ return parseWheelFilename(strings.TrimSuffix(filename, ".whl"))
+ case strings.HasSuffix(filename, ".egg"):
+ return parseEggFilename(strings.TrimSuffix(filename, ".egg"))
+ }
+
+ for _, ext := range windowsInstallerExtensions {
+ if strings.HasSuffix(filename, ext) {
+ return parseWindowsInstallerFilename(strings.TrimSuffix(filename, ext))
+ }
+ }
+
+ for _, ext := range archiveExtensions {
+ if strings.HasSuffix(filename, ext) {
+ return splitNameVersion(strings.TrimSuffix(filename, ext))
+ }
+ }
+
+ return "", ""
+}
+
+// parseWheelFilename parses the PEP 427 layout
+// {name}-{version}(-{build})?-{python}-{abi}-{platform}, base being the
+// filename without its ".whl" suffix. The spec escapes every hyphen in the name
+// and version to '_', so the first two fields are authoritative even when the
+// optional build tag is present.
+func parseWheelFilename(base string) (name, version string) {
+ parts := strings.Split(base, "-")
+ if len(parts) < minWheelParts {
+ return "", ""
+ }
+
+ return parts[0], parts[1]
+}
+
+// parseEggFilename parses the setuptools bdist_egg layout
+// {name}-{version}-py{X.Y}(-{platform})?, base being the filename without its
+// ".egg" suffix. setuptools escapes hyphens in the name and version to '_', but
+// eggs built by other tooling do not always, so the version is located relative
+// to the interpreter field rather than assumed to be the second field.
+func parseEggFilename(base string) (name, version string) {
+ parts := strings.Split(base, "-")
+ // Scan from the end: the trailing platform fields never look like an
+ // interpreter tag, so the last match is the real one even when the package
+ // name itself carries a "py{N}" component. Stop before index 1, since a tag
+ // any earlier would leave no room for both a name and a version.
+ for i := len(parts) - 1; i >= minEggParts-1; i-- {
+ if !isEggPythonTag(parts[i]) || !isVersionField(parts[i-1]) {
+ continue
+ }
+
+ return strings.Join(parts[:i-1], "-"), parts[i-1]
+ }
+
+ // No interpreter field: {name}-{version}.
+ return splitNameVersion(base)
+}
+
+// parseWindowsInstallerFilename parses the distutils bdist_wininst and
+// bdist_msi layout {name}-{version}.{platform}(-py{X.Y})?, base being the
+// filename without its ".exe" or ".msi" suffix. The platform is joined to the
+// version with a '.' rather than a '-' and may itself contain a hyphen
+// ("win-amd64"), so both trailing fields are stripped before the name and
+// version are split apart.
+func parseWindowsInstallerFilename(base string) (name, version string) {
+ if i := strings.LastIndex(base, "-py"); i >= 0 && isDottedNumber(base[i+len("-py"):]) {
+ base = base[:i]
+ }
+
+ // The platform is the final '.'-separated field. Requiring it to start with
+ // a non-digit keeps a dotted version from being truncated when a filename
+ // carries no platform tag.
+ i := strings.LastIndex(base, ".")
+ if i < 0 || i+1 >= len(base) || isVersionStart(base[i+1]) {
+ return "", ""
+ }
+
+ return splitFullname(base[:i])
+}
+
+// splitFullname splits the distutils fullname {name}-{version} that precedes a
+// Windows installer's platform field. Unlike an sdist, a wininst fullname may
+// carry a trailing build variant ("cx_Oracle-5.1.2-11g"), which belongs to
+// neither the name nor the version, so the first purely numeric field wins and
+// anything after it is discarded.
+func splitFullname(fullname string) (name, version string) {
+ parts := strings.Split(fullname, "-")
+ for i := 1; i < len(parts); i++ {
+ if isDottedNumber(parts[i]) {
+ return strings.Join(parts[:i], "-"), parts[i]
+ }
+ }
+
+ // No purely numeric field, e.g. a prerelease version like "1.0b1".
+ return splitNameVersion(fullname)
+}
+
+// splitNameVersion splits a {name}-{version} pair at the last hyphen that
+// starts a version, leaving hyphens inside the name intact.
+func splitNameVersion(base string) (name, version string) {
+ for i := len(base) - 1; i >= 0; i-- {
+ if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) {
+ return base[:i], base[i+1:]
+ }
+ }
+
+ return "", ""
+}
+
+// isEggPythonTag reports whether field is the py{X.Y} interpreter field that
+// setuptools places directly after the version in an egg filename.
+func isEggPythonTag(field string) bool {
+ const prefix = "py"
+
+ return len(field) > len(prefix) && strings.HasPrefix(field, prefix) && isVersionStart(field[len(prefix)])
+}
+
+// isVersionField reports whether field can be a version, i.e. it is non-empty
+// and starts with a digit as every PEP 440 release segment does.
+func isVersionField(field string) bool {
+ return field != "" && isVersionStart(field[0])
+}
+
+// isDottedNumber reports whether s is a dotted numeric version such as "2.7".
+func isDottedNumber(s string) bool {
+ if s == "" || !isVersionStart(s[0]) {
+ return false
+ }
+
+ for i := range len(s) {
+ if !isVersionStart(s[i]) && s[i] != '.' {
+ return false
+ }
+ }
+
+ return true
+}
+
+func isVersionStart(c byte) bool {
+ return c >= '0' && c <= '9'
+}
diff --git a/internal/metadata/pypi/imports_test.go b/internal/metadata/pypi/imports_test.go
new file mode 100644
index 00000000..3a576f83
--- /dev/null
+++ b/internal/metadata/pypi/imports_test.go
@@ -0,0 +1,45 @@
+package pypi
+
+import (
+ "os/exec"
+ "slices"
+ "strings"
+ "testing"
+)
+
+// TestPortableImports keeps the package free of the proxy's infrastructure:
+// no handler, database, storage, metrics, HTTP or logging, so it builds for
+// targets such as WASI where those are not available.
+func TestPortableImports(t *testing.T) {
+ goTool, err := exec.LookPath("go")
+ if err != nil {
+ t.Skip("go tool not found")
+ }
+ out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output()
+ if err != nil {
+ t.Fatalf("go list: %v", err)
+ }
+ allowed := []string{
+ "github.com/git-pkgs/cooldown",
+ "github.com/git-pkgs/proxy/internal/metadata/pypi",
+ "github.com/git-pkgs/proxy/internal/packageurl",
+ "github.com/git-pkgs/purl",
+ "github.com/git-pkgs/vers",
+ "github.com/package-url/packageurl-go",
+ "golang.org/x/net/html",
+ "golang.org/x/net/html/atom",
+ }
+ forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"}
+ for line := range strings.Lines(string(out)) {
+ standard, path, _ := strings.Cut(strings.TrimSpace(line), " ")
+ if standard == "true" {
+ if slices.Contains(forbiddenStd, path) {
+ t.Errorf("imports %s", path)
+ }
+ continue
+ }
+ if !slices.Contains(allowed, path) {
+ t.Errorf("imports %s, which is not on the portable list", path)
+ }
+ }
+}
diff --git a/internal/metadata/pypi/metadata.go b/internal/metadata/pypi/metadata.go
new file mode 100644
index 00000000..ca5f01e3
--- /dev/null
+++ b/internal/metadata/pypi/metadata.go
@@ -0,0 +1,421 @@
+// Package pypi transforms PyPI metadata without storage or network access.
+package pypi
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "maps"
+ "net/url"
+ "path"
+ "regexp"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/git-pkgs/cooldown"
+ "github.com/git-pkgs/proxy/internal/packageurl"
+ "golang.org/x/net/html"
+)
+
+const minSubmatchParts = 2 // full match + first capture group
+
+// CooldownPolicy evaluates release age at the caller's supplied time.
+type CooldownPolicy interface {
+ Enabled() bool
+ Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision
+}
+
+// Options supplies the public and download URLs, the policy and the callbacks
+// for a rewrite. EvaluatedAt is used as given; the transformation never reads
+// the clock.
+type Options struct {
+ // ProxyURL is the proxy's public URL; DownloadURL is the host files are
+ // served from upstream, such as https://files.pythonhosted.org.
+ ProxyURL string
+ DownloadURL string
+ Cooldown CooldownPolicy
+ EvaluatedAt time.Time
+ // Denied reports whether the denylist names a version of a package.
+ Denied func(name, version string) bool
+ // Optional callbacks report changes synchronously. OnFiltered is called
+ // for each release the JSON API rewrite leaves out, OnWithheld for each
+ // of those that cooldown held back, with its newest upload time.
+ OnFiltered func(name, version string)
+ OnWithheld func(name, version string, publishedAt time.Time)
+}
+
+var (
+ // simpleLinkRe matches a whole anchor tag: filename.
+ simpleLinkRe = regexp.MustCompile(`]+href="[^"]*"[^>]*>[^<]+`)
+ // simpleLinkTextRe captures an anchor's text.
+ simpleLinkTextRe = regexp.MustCompile(`>([^<]+)`)
+)
+
+// RewriteSimpleHTML rewrites package URLs in simple API HTML to point at the
+// proxy. Links for the versions in filteredVersions are removed entirely.
+func RewriteSimpleHTML(body []byte, filteredVersions map[string]bool, o Options) []byte {
+ if len(filteredVersions) > 0 {
+ body = simpleLinkRe.ReplaceAllFunc(body, func(match []byte) []byte {
+ innerMatch := simpleLinkTextRe.FindSubmatch(match)
+ if len(innerMatch) < minSubmatchParts {
+ return match
+ }
+ _, version := FilenameIdentity(strings.TrimSpace(string(innerMatch[1])))
+ if version != "" && filteredVersions[version] {
+ return nil
+ }
+ return match
+ })
+ }
+ return RewriteDownloadHrefs(body, o)
+}
+
+// RewriteDownloadHrefs points href attributes for files on the download host
+// at the proxy. It makes the same replacements as the regular expression
+// `href="(/packages/[^"]+)"` would: an href whose URL starts
+// with the download host's /packages/ and has at least one more character
+// before the closing quote. Finding the literal prefix is many times faster
+// than running the regular expression over every link.
+func RewriteDownloadHrefs(body []byte, o Options) []byte {
+ from := []byte(`href="` + o.DownloadURL + `/packages/`)
+ // The download route is /pypi/packages/{path...} and keeps the path
+ // from the download host, /packages/ included.
+ to := []byte(`href="` + o.ProxyURL + `/pypi/packages/packages/`)
+
+ var out []byte
+ done := 0 // body[:done] is in out already
+ for search := 0; ; {
+ i := bytes.Index(body[search:], from)
+ if i < 0 {
+ break
+ }
+ start := search + i
+ rest := start + len(from)
+ end := bytes.IndexByte(body[rest:], '"')
+ if end <= 0 {
+ // No closing quote, or nothing before it: not a match. The
+ // regular expression would go on looking from the next byte.
+ search = start + 1
+ continue
+ }
+ if out == nil {
+ out = make([]byte, 0, len(body)+len(body)/8)
+ }
+ out = append(out, body[done:start]...)
+ out = append(out, to...)
+ done = rest
+ search = rest + end + 1
+ }
+ if out == nil {
+ return body
+ }
+ return append(out, body[done:]...)
+}
+
+// FilterSimpleHTMLLinks removes the links to files of the given versions,
+// read from their hrefs rather than their display text. Tokenization handles
+// single quotes, entity escaping and nested link text without reserializing
+// the rest of the index.
+func FilterSimpleHTMLLinks(body []byte, versions map[string]bool) []byte {
+ if len(versions) == 0 {
+ return body
+ }
+ var result bytes.Buffer
+ z := html.NewTokenizer(bytes.NewReader(body))
+ skip := false
+ for {
+ tokenType := z.Next()
+ if tokenType == html.ErrorToken {
+ return result.Bytes()
+ }
+ raw := append([]byte(nil), z.Raw()...)
+ if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
+ token := z.Token()
+ if token.Data == "a" {
+ skip = deniedSimpleLink(token, versions)
+ }
+ } else if tokenType == html.EndTagToken && z.Token().Data == "a" {
+ if skip {
+ skip = false
+ continue
+ }
+ }
+ if !skip {
+ result.Write(raw)
+ }
+ }
+}
+
+func deniedSimpleLink(token html.Token, versions map[string]bool) bool {
+ for _, attr := range token.Attr {
+ if attr.Key != "href" {
+ continue
+ }
+ u, err := url.Parse(attr.Val)
+ if err != nil {
+ continue
+ }
+ _, version := FilenameIdentity(path.Base(u.Path))
+ if versions[version] {
+ return true
+ }
+ }
+ return false
+}
+
+// RewriteSimpleJSON rewrites a PEP 691 simple API document: file URLs point
+// at the proxy, and the files and versions of filteredVersions are left out.
+func RewriteSimpleJSON(body []byte, filteredVersions map[string]bool, o Options) ([]byte, error) {
+ var metadata map[string]any
+ if err := json.Unmarshal(body, &metadata); err != nil {
+ return nil, err
+ }
+
+ files, ok := metadata["files"].([]any)
+ if !ok {
+ return nil, errors.New("pypi simple json response has no files array")
+ }
+
+ rewrittenFiles := make([]any, 0, len(files))
+ for _, file := range files {
+ entry, ok := file.(map[string]any)
+ if !ok {
+ rewrittenFiles = append(rewrittenFiles, file)
+ continue
+ }
+
+ if filename, ok := entry["filename"].(string); ok {
+ _, version := FilenameIdentity(filename)
+ if version != "" && filteredVersions[version] {
+ continue
+ }
+ }
+
+ rewriteURLEntry(entry, o)
+ rewrittenFiles = append(rewrittenFiles, entry)
+ }
+
+ metadata["files"] = rewrittenFiles
+
+ // The versions list (PEP 700) loses the filtered versions too, whose
+ // files were all left out above.
+ if versions, ok := metadata["versions"].([]any); ok && len(filteredVersions) > 0 {
+ keptVersions := make([]any, 0, len(versions))
+ for _, v := range versions {
+ if version, ok := v.(string); ok && filteredVersions[version] {
+ continue
+ }
+ keptVersions = append(keptVersions, v)
+ }
+ metadata["versions"] = keptVersions
+ }
+ return json.Marshal(metadata)
+}
+
+// RewriteJSON rewrites a JSON API document, a package's or one version's:
+// releases the denylist names or cooldown holds back are left out, and file
+// URLs point at the proxy. It also returns what it kept: the releases, and
+// the current version's files unless they were dropped.
+func RewriteJSON(body []byte, o Options) ([]byte, []string, error) {
+ var metadata map[string]any
+ if err := json.Unmarshal(body, &metadata); err != nil {
+ return nil, nil, err
+ }
+
+ packageName, _ := documentName(metadata)
+ packagePURL := ""
+ if packageName != "" {
+ packagePURL = packageurl.MakeString("pypi", packageName, "")
+ }
+
+ filterAndRewriteReleases(metadata, packageName, packagePURL, o)
+ filterAndRewriteURLs(metadata, packagePURL, o)
+
+ var kept []string
+ if releases, ok := metadata["releases"].(map[string]any); ok {
+ kept = slices.Collect(maps.Keys(releases))
+ }
+ if urls, ok := metadata["urls"].([]any); ok && len(urls) > 0 {
+ // Not a valid release key, so it can't be confused with one.
+ kept = append(kept, "\x00urls")
+ }
+
+ out, err := json.Marshal(metadata)
+ return out, kept, err
+}
+
+// filterAndRewriteReleases applies the denylist and cooldown to the releases
+// map and rewrites the URLs of the files it keeps.
+func filterAndRewriteReleases(metadata map[string]any, packageName, packagePURL string, o Options) {
+ releases, ok := metadata["releases"].(map[string]any)
+ if !ok {
+ return
+ }
+
+ for version, files := range releases {
+ denied := o.denied(packageName, version)
+ held, publishedAt := false, time.Time{}
+ if !denied {
+ held, publishedAt = heldBack(packagePURL, files, o)
+ }
+ if denied || held {
+ if o.OnFiltered != nil {
+ o.OnFiltered(packageName, version)
+ }
+ if held && o.OnWithheld != nil {
+ o.OnWithheld(packageName, version, publishedAt)
+ }
+ delete(releases, version)
+ continue
+ }
+
+ rewriteFileEntries(files, o)
+ }
+}
+
+// filterAndRewriteURLs applies the denylist and cooldown to the urls array,
+// the current version's files, and rewrites the URLs it keeps.
+func filterAndRewriteURLs(metadata map[string]any, packagePURL string, o Options) {
+ urls, ok := metadata["urls"].([]any)
+ if !ok {
+ return
+ }
+
+ name, version := documentNameVersion(metadata)
+ if o.denied(name, version) {
+ metadata["urls"] = []any{}
+ } else if held, publishedAt := heldBack(packagePURL, urls, o); held {
+ if o.OnWithheld != nil {
+ o.OnWithheld(name, version, publishedAt)
+ }
+ metadata["urls"] = []any{}
+ }
+
+ if urls, ok := metadata["urls"].([]any); ok {
+ for _, u := range urls {
+ if umap, ok := u.(map[string]any); ok {
+ rewriteURLEntry(umap, o)
+ }
+ }
+ }
+}
+
+func (o Options) denied(name, version string) bool {
+ return o.Denied != nil && o.Denied(name, version)
+}
+
+// heldBack reports whether cooldown holds a release back, and the newest
+// upload time that decided it.
+func heldBack(packagePURL string, files any, o Options) (bool, time.Time) {
+ if o.Cooldown == nil || !o.Cooldown.Enabled() || packagePURL == "" {
+ return false, time.Time{}
+ }
+ filesArr, ok := files.([]any)
+ if !ok {
+ return false, time.Time{}
+ }
+ publishedAt := newestUploadTime(filesArr)
+ return !publishedAt.IsZero() && !o.Cooldown.Evaluate("pypi", packagePURL, publishedAt, o.EvaluatedAt).Allowed, publishedAt
+}
+
+func rewriteFileEntries(files any, o Options) {
+ filesArr, ok := files.([]any)
+ if !ok {
+ return
+ }
+ for _, f := range filesArr {
+ if fmap, ok := f.(map[string]any); ok {
+ rewriteURLEntry(fmap, o)
+ }
+ }
+}
+
+// documentName returns the package name a JSON API document gives.
+func documentName(metadata map[string]any) (string, bool) {
+ info, ok := metadata["info"].(map[string]any)
+ if !ok {
+ return "", false
+ }
+ name, ok := info["name"].(string)
+ return name, ok
+}
+
+func documentNameVersion(metadata map[string]any) (name, version string) {
+ info, _ := metadata["info"].(map[string]any)
+ name, _ = info["name"].(string)
+ version, _ = info["version"].(string)
+ return name, version
+}
+
+// newestUploadTime returns the most recent upload_time_iso_8601 of a list of
+// file entries.
+func newestUploadTime(files []any) time.Time {
+ var newest time.Time
+ for _, f := range files {
+ fmap, ok := f.(map[string]any)
+ if !ok {
+ continue
+ }
+ ts, ok := fmap["upload_time_iso_8601"].(string)
+ if !ok {
+ continue
+ }
+ t, err := time.Parse(time.RFC3339, ts)
+ if err != nil {
+ continue
+ }
+ if t.After(newest) {
+ newest = t
+ }
+ }
+ return newest
+}
+
+// rewriteURLEntry points a file entry's URL on the download host at the proxy.
+func rewriteURLEntry(entry map[string]any, o Options) {
+ urlStr, ok := entry["url"].(string)
+ if !ok {
+ return
+ }
+ if strings.HasPrefix(urlStr, o.DownloadURL+"/packages/") {
+ entry["url"] = o.ProxyURL + "/pypi/packages" + strings.TrimPrefix(urlStr, o.DownloadURL)
+ }
+}
+
+// releaseFiles is the part of a JSON API document the release times are
+// read from. Decoding into it skips every other field, where a generic
+// map would hold the whole document, several times its size, in memory.
+type releaseFiles struct {
+ Releases map[string][]struct {
+ UploadTime string `json:"upload_time_iso_8601"`
+ } `json:"releases"`
+}
+
+// ReleaseTimes returns each release's newest upload time from a JSON API
+// document, zero for a release without one.
+func ReleaseTimes(body []byte) (map[string]time.Time, error) {
+ var metadata releaseFiles
+ // A value of another type than expected is skipped and the rest is
+ // still decoded, so a file entry without a usable time doesn't count.
+ var mismatch *json.UnmarshalTypeError
+ if err := json.Unmarshal(body, &metadata); err != nil && !errors.As(err, &mismatch) {
+ return nil, fmt.Errorf("parsing pypi json metadata: %w", err)
+ }
+ if metadata.Releases == nil {
+ return nil, errors.New("pypi json metadata has no releases")
+ }
+
+ times := make(map[string]time.Time, len(metadata.Releases))
+ for version, files := range metadata.Releases {
+ var newest time.Time
+ for _, file := range files {
+ if t, err := time.Parse(time.RFC3339, file.UploadTime); err == nil && t.After(newest) {
+ newest = t
+ }
+ }
+ times[version] = newest
+ }
+ return times, nil
+}
diff --git a/internal/metadata/pypi/metadata_test.go b/internal/metadata/pypi/metadata_test.go
new file mode 100644
index 00000000..de629440
--- /dev/null
+++ b/internal/metadata/pypi/metadata_test.go
@@ -0,0 +1,182 @@
+package pypi
+
+import (
+ "encoding/json"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/git-pkgs/cooldown"
+)
+
+// fixedCooldown holds back releases published within a day of the time a
+// rewrite is evaluated at, so tests can tell that time from the clock.
+type fixedCooldown struct{}
+
+func (fixedCooldown) Enabled() bool { return true }
+
+func (fixedCooldown) Evaluate(_, _ string, publishedAt, evaluatedAt time.Time) cooldown.Decision {
+ return cooldown.Decision{Allowed: evaluatedAt.Sub(publishedAt) >= 24*time.Hour}
+}
+
+const jsonDocument = `{
+ "info": {"name": "Demo", "version": "3.0.0"},
+ "releases": {
+ "1.0.0": [{"url": "https://files.example/packages/aa/demo-1.0.0.tar.gz", "upload_time_iso_8601": "2026-01-01T00:00:00Z"}],
+ "2.0.0": [{"url": "https://files.example/packages/bb/demo-2.0.0.tar.gz", "upload_time_iso_8601": "2026-01-10T00:00:00Z"}],
+ "3.0.0": [
+ {"url": "https://files.example/packages/cc/demo-3.0.0.tar.gz", "upload_time_iso_8601": "2026-01-09T00:00:00Z"},
+ {"url": "https://files.example/packages/dd/demo-3.0.0-py3-none-any.whl", "upload_time_iso_8601": "2026-01-10T12:00:00Z"}
+ ]
+ },
+ "urls": [{"url": "https://files.example/packages/dd/demo-3.0.0-py3-none-any.whl", "upload_time_iso_8601": "2026-01-10T12:00:00Z"}]
+}`
+
+func testOptions() Options {
+ return Options{
+ ProxyURL: "https://proxy.example",
+ DownloadURL: "https://files.example",
+ Cooldown: fixedCooldown{},
+ EvaluatedAt: time.Date(2026, 1, 11, 0, 0, 0, 0, time.UTC),
+ }
+}
+
+// Cooldown is decided at EvaluatedAt, by a release's newest file; the
+// denylist leaves out what it names, and both are reported.
+func TestRewriteJSON(t *testing.T) {
+ o := testOptions()
+ o.Denied = func(name, version string) bool { return name == "Demo" && version == "1.0.0" }
+ var filtered, withheld []string
+ var withheldAt time.Time
+ o.OnFiltered = func(name, version string) { filtered = append(filtered, name+"@"+version) }
+ o.OnWithheld = func(name, version string, at time.Time) {
+ withheld = append(withheld, name+"@"+version)
+ if version == "3.0.0" {
+ withheldAt = at
+ }
+ }
+
+ out, kept, err := RewriteJSON([]byte(jsonDocument), o)
+ if err != nil {
+ t.Fatalf("RewriteJSON: %v", err)
+ }
+ slices.Sort(kept)
+ if !slices.Equal(kept, []string{"2.0.0"}) {
+ t.Errorf("kept = %q, want only 2.0.0", kept)
+ }
+ slices.Sort(filtered)
+ if !slices.Equal(filtered, []string{"Demo@1.0.0", "Demo@3.0.0"}) {
+ t.Errorf("filtered = %v", filtered)
+ }
+ // 3.0.0 twice: as a release, and as the urls of the current version.
+ if !slices.Equal(withheld, []string{"Demo@3.0.0", "Demo@3.0.0"}) {
+ t.Errorf("withheld = %v", withheld)
+ }
+ if want := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC); !withheldAt.Equal(want) {
+ t.Errorf("withheld at %v, want the newest file's %v", withheldAt, want)
+ }
+
+ var doc struct {
+ Releases map[string][]struct{ URL string } `json:"releases"`
+ URLs []any `json:"urls"`
+ }
+ if err := json.Unmarshal(out, &doc); err != nil {
+ t.Fatal(err)
+ }
+ if got := doc.Releases["2.0.0"][0].URL; got != "https://proxy.example/pypi/packages/packages/bb/demo-2.0.0.tar.gz" {
+ t.Errorf("2.0.0 URL = %q", got)
+ }
+ if len(doc.URLs) != 0 {
+ t.Errorf("urls = %v, want them dropped with 3.0.0", doc.URLs)
+ }
+}
+
+// Without a policy or a denylist everything is kept and only URLs change.
+func TestRewriteJSONWithoutPolicy(t *testing.T) {
+ o := Options{ProxyURL: "https://proxy.example", DownloadURL: "https://files.example"}
+ out, kept, err := RewriteJSON([]byte(jsonDocument), o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(kept) != 4 {
+ t.Errorf("kept = %q, want three releases and the urls", kept)
+ }
+ if strings.Contains(string(out), "https://files.example") {
+ t.Errorf("a download URL was left: %s", out)
+ }
+}
+
+func TestRewriteSimpleHTMLAndJSON(t *testing.T) {
+ o := testOptions()
+ html := `demo-1.0.0.tar.gz
+demo-2.0.0.tar.gz`
+ got := string(RewriteSimpleHTML(FilterSimpleHTMLLinks([]byte(html), map[string]bool{"1.0.0": true}), map[string]bool{"1.0.0": true}, o))
+ if strings.Contains(got, "1.0.0") || !strings.Contains(got, `href="https://proxy.example/pypi/packages/packages/bb/demo-2.0.0.tar.gz#sha256=2"`) {
+ t.Errorf("simple HTML = %q", got)
+ }
+
+ simple := `{"files": [
+ {"filename": "demo-1.0.0.tar.gz", "url": "https://files.example/packages/aa/demo-1.0.0.tar.gz"},
+ {"filename": "demo-2.0.0.tar.gz", "url": "https://files.example/packages/bb/demo-2.0.0.tar.gz"}],
+ "versions": ["1.0.0", "2.0.0"]}`
+ out, err := RewriteSimpleJSON([]byte(simple), map[string]bool{"1.0.0": true}, o)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var doc struct {
+ Files []struct{ URL string } `json:"files"`
+ Versions []string `json:"versions"`
+ }
+ if err := json.Unmarshal(out, &doc); err != nil {
+ t.Fatal(err)
+ }
+ if len(doc.Files) != 1 || doc.Files[0].URL != "https://proxy.example/pypi/packages/packages/bb/demo-2.0.0.tar.gz" ||
+ !slices.Equal(doc.Versions, []string{"2.0.0"}) {
+ t.Errorf("simple JSON = %s", out)
+ }
+}
+
+func TestReleaseTimes(t *testing.T) {
+ times, err := ReleaseTimes([]byte(jsonDocument))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if want := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC); !times["3.0.0"].Equal(want) {
+ t.Errorf("3.0.0 = %v, want the newest file's %v", times["3.0.0"], want)
+ }
+}
+
+func TestFilenameIdentity(t *testing.T) {
+ for filename, want := range map[string][2]string{
+ "requests-2.31.0-py3-none-any.whl": {"requests", "2.31.0"},
+ "requests-2.31.0.tar.gz": {"requests", "2.31.0"},
+ "requests-2.31.0-py3-none-any.whl.metadata": {"requests", "2.31.0"},
+ "numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg": {"numpy", "1.8.0"},
+ "numpy-1.8.0.win32-py2.7.exe": {"numpy", "1.8.0"},
+ "README.txt": {"", ""},
+ } {
+ if name, version := FilenameIdentity(filename); name != want[0] || version != want[1] {
+ t.Errorf("FilenameIdentity(%q) = %q, %q; want %q, %q", filename, name, version, want[0], want[1])
+ }
+ }
+}
+
+// disabledCooldown would hold back everything, but is off.
+type disabledCooldown struct{}
+
+func (disabledCooldown) Enabled() bool { return false }
+
+func (disabledCooldown) Evaluate(_, _ string, _, _ time.Time) cooldown.Decision {
+ return cooldown.Decision{Allowed: false}
+}
+
+// A policy that is turned off holds nothing back.
+func TestRewriteJSONWithDisabledCooldown(t *testing.T) {
+ o := testOptions()
+ o.Cooldown = disabledCooldown{}
+ _, kept, err := RewriteJSON([]byte(jsonDocument), o)
+ if err != nil || len(kept) != 4 {
+ t.Errorf("kept = %q, %v; want every release and the urls", kept, err)
+ }
+}
diff --git a/internal/mirror/files.go b/internal/mirror/files.go
index ab6bafdb..a4109765 100644
--- a/internal/mirror/files.go
+++ b/internal/mirror/files.go
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/git-pkgs/proxy/internal/handler"
+ "github.com/git-pkgs/proxy/internal/metadata/pypi"
"github.com/git-pkgs/registries"
)
@@ -39,7 +40,7 @@ var fileEcosystems = map[string]fileEcosystem{
return map[string]string{"file_name": file.Filename}
},
cacheKey: func(pv PackageVersion, file registries.Artifact) (string, string, string, bool) {
- name, version := handler.PyPIFilenameIdentity(file.Filename)
+ name, version := pypi.FilenameIdentity(file.Filename)
if name == "" {
name, version = pv.Name, pv.Version
}