diff --git a/docs/configuration.md b/docs/configuration.md index e7305c40..89257564 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1325,7 +1325,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. When a metadata filter holds a version back, the proxy stores its publish time, so the version is listed even if nobody tried to download it, and a package with no downloads yet gets a page too. Conda and Helm don't record these versions: a conda build has a time of its own, and Helm charts are stored per repository. Neither do a Maven named repository and routed npm and Composer packages, whose publish times are kept apart from the default registry's, nor a Go module with capitals in its path, whose times are stored under the escaped path. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 8a2e0574..398da071 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1245,5 +1245,120 @@ func TestVersionPublishedAtIsUTC(t *testing.T) { t.Fatal(err) } check("pkg:cargo/demo@2.0.0") + + if err := db.RecordWithheldVersion(&Package{PURL: "pkg:cargo/demo", Ecosystem: "cargo", Name: "demo"}, + "pkg:cargo/demo@3.0.0", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@3.0.0") + }) +} + +// A version a cooldown filter holds back is recorded with its publish time, +// and its package gets a row so the package page exists before anything is +// downloaded. Rows already there keep what they have. +func TestRecordWithheldVersion(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + published := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@2.0.0", published); err != nil { + t.Fatalf("RecordWithheldVersion: %v", err) + } + got, err := db.GetPackageByEcosystemName("npm", "leftpad") + if err != nil || got == nil { + t.Fatalf("package row = %v, %v", got, err) + } + if got.PURL != pkgPURL { + t.Errorf("package purl = %q, want %q", got.PURL, pkgPURL) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil { + t.Fatalf("version row = %v, %v", ver, err) + } + if ver.PackagePURL != pkgPURL || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version = %+v, want package %s published %v", ver, pkgPURL, published) + } + }) +} + +// For an ecosystem whose release time is its newest file's, a later time +// replaces the stored one and an earlier one doesn't. +func TestRecordWithheldVersionNewest(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:pypi/ruff" + first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + later := first.Add(time.Hour) + pkg := &Package{PURL: pkgPURL, Ecosystem: "pypi", Name: "ruff"} + for _, at := range []time.Time{first, later, first} { + if err := db.RecordWithheldVersionNewest(pkg, pkgPURL+"@2.0.0", at); err != nil { + t.Fatalf("RecordWithheldVersionNewest: %v", err) + } + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) { + t.Errorf("version = %+v, %v; want published %v", ver, err, later) + } + if got, err := db.GetPackageByEcosystemName("pypi", "ruff"); err != nil || got == nil { + t.Errorf("package row = %v, %v", got, err) + } + }) +} + +// AdvanceVersionPublishedAt only moves a stored time forward, and creates +// the row when there is none. +func TestAdvanceVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:pypi/ruff" + first := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + later := first.Add(time.Hour) + for _, at := range []time.Time{first, later, first} { + if err := db.AdvanceVersionPublishedAt(pkgPURL+"@2.0.0", pkgPURL, at); err != nil { + t.Fatalf("AdvanceVersionPublishedAt: %v", err) + } + } + ver, err := db.GetVersionByPURL(pkgPURL + "@2.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(later) { + t.Errorf("version = %+v, %v; want published %v", ver, err, later) + } + }) +} + +func TestRecordWithheldVersionKeepsExistingRows(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkgPURL = "pkg:npm/leftpad" + stored := time.Date(2026, 11, 1, 8, 0, 0, 0, time.UTC) + other := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad", + Description: sql.NullString{String: "pads left", Valid: true}}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.SetVersionPublishedAt(pkgPURL+"@1.0.0", pkgPURL, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: pkgPURL + "@3.0.0", PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + pkg := &Package{PURL: pkgPURL, Ecosystem: "npm", Name: "leftpad"} + for _, v := range []string{"1.0.0", "3.0.0"} { + if err := db.RecordWithheldVersion(pkg, pkgPURL+"@"+v, other); err != nil { + t.Fatalf("RecordWithheldVersion %s: %v", v, err) + } + } + + got, err := db.GetPackageByPURL(pkgPURL) + if err != nil || got == nil || got.Description.String != "pads left" { + t.Errorf("package = %+v, %v; want the description kept", got, err) + } + ver, err := db.GetVersionByPURL(pkgPURL + "@1.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Time.Equal(stored) { + t.Errorf("1.0.0 = %+v, %v; want the stored time %v kept", ver, err, stored) + } + ver, err = db.GetVersionByPURL(pkgPURL + "@3.0.0") + if err != nil || ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(other) { + t.Errorf("3.0.0 = %+v, %v; want the empty time filled with %v", ver, err, other) + } }) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 6f693178..10cb43d3 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -225,6 +225,76 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// AdvanceVersionPublishedAt is SetVersionPublishedAt for a time that only +// moves forward, such as a PyPI release's newest upload: an earlier time, +// read from an older document, leaves the stored one. +func (db *DB) AdvanceVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + _, err := db.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL OR versions.published_at < excluded.published_at + `), versionPURL, packagePURL, publishedAt.UTC(), now, now) + if err != nil { + return fmt.Errorf("advancing version publish time: %w", err) + } + return nil +} + +// RecordWithheldVersion records a version a cooldown filter held back: its +// publish time, and a row for its package so the package page exists before +// anything is downloaded. It only adds what is missing. An existing package +// row is left as it is, and a stored publish time is not overwritten, so the +// filters can call it for every version they hold back without reading +// first. +func (db *DB) RecordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, "versions.published_at IS NULL") +} + +// RecordWithheldVersionNewest is RecordWithheldVersion for an ecosystem whose +// release time is its newest file's, such as PyPI: a later time replaces the +// stored one, since a file can be added to a release after it was held back. +func (db *DB) RecordWithheldVersionNewest(pkg *Package, versionPURL string, publishedAt time.Time) error { + return db.recordWithheldVersion(pkg, versionPURL, publishedAt, + "versions.published_at IS NULL OR versions.published_at < excluded.published_at") +} + +// recordWithheldVersion records a held back version, updating the stored +// publish time where replace, a condition on the existing row, holds. +func (db *DB) recordWithheldVersion(pkg *Package, versionPURL string, publishedAt time.Time, replace string) error { + now := time.Now() + tx, err := db.Beginx() + if err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.Exec(db.Rebind(` + INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO NOTHING + `), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now); err != nil { + return fmt.Errorf("recording withheld version's package: %w", err) + } + if _, err := tx.Exec(db.Rebind(` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE `+replace+` + `), versionPURL, pkg.PURL, publishedAt.UTC(), now, now); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("recording withheld version: %w", err) + } + return nil +} + // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c742145..337c8031 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -180,6 +180,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) + h.proxy.recordWithheld("cargo", crate.Name, crate.Version, publishedAt) continue } diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 6b82cfff..1fccb2ac 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -1,19 +1,16 @@ package handler import ( - "bytes" "context" "encoding/json" "errors" "fmt" "net/http" - "net/url" "path" - "regexp" "strings" "time" - "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/metadata/composer" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -21,15 +18,10 @@ import ( const ( composerUpstream = "https://packagist.org" composerRepo = "https://repo.packagist.org" - composerMinified = "composer/2.0" - composerUnset = "__unset" - composerDevReset = "~dev" composerDevFile = "~dev" // suffix of the metadata file holding dev versions vendorPackageParts = 2 ) -var composerUnsetJSON = []byte(`"` + composerUnset + `"`) - // Archives are stored under pkg:composer/{vendor}/{package}, which purl // lowercases, so a package rule matches whatever case it is written in. The // spec also claims the "packagist" spelling git-pkgs writes into @@ -224,406 +216,53 @@ func (h *ComposerHandler) rewriteMetadataKeeping(body []byte) ([]byte, []string, } // rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the -// pushed versions in local under localPackage, in place of any upstream entry -// with the same version. Cooldown only filters upstream versions. A document -// that does not list localPackage at all is replaced by one holding only the -// pushed versions. +// pushed versions in local under localPackage; see composer.Rewrite. func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage string, local []composerLocalEntry) ([]byte, []string, error) { - if !json.Valid(body) { - return nil, nil, errors.New("composer metadata is not valid JSON") - } - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return nil, nil, err - } - minified := format == composerMinified - - var out bytes.Buffer - var kept []string - listed := false - out.Grow(len(body) + len(body)/8) - err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { - packages := m.Value(body) - if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { - return false, nil - } - return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { - versions := p.Value(packages) - if versions[0] != '[' { - return false, nil - } - packageName, err := jsonscan.Key(p.Key(packages)) - if err != nil { - return false, err - } - var pushed []composerLocalEntry - if packageName == localPackage { - pushed, listed = local, true - } - return true, h.writeVersions(out, packageName, versions, minified, &kept, pushed) - }) - }) - if err != nil { - return nil, nil, err - } - if len(local) > 0 && !listed { - document, err := localComposerDocument(localPackage, local) - return document, nil, err - } - return out.Bytes(), kept, nil -} - -// writeVersions writes one package's version list without the versions in -// cooldown and with each dist pointing at this proxy, and appends each version -// it writes to kept as its package name and version. -// -// Minified lists stay minified. Each version written carries the fields that -// differ from what the client has expanded so far, which is the upstream -// entry itself unless filtered versions were dropped before it; then it also -// carries the changes those versions made. Every version gets its own dist, -// since the proxied URL names the version. -// -// The pushed versions in local follow the upstream ones and take the place of -// upstream entries with the same version. -func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []composerLocalEntry) error { - packagePURL := canonicalPackagePURL("composer", packageName) - upstream, written := newComposerFields(), newComposerFields() - devReset, first := false, true - _, routed := h.routes.match(packageName) - - out.WriteByte('[') - err := jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - upstream.reset() - written.reset() - devReset = true - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - upstream.reset() - written.reset() - } - if err := upstream.apply(entry, minified); err != nil { - return err - } - - version := upstream.string("version") - if pushedComposerVersion(local, version) || h.shouldFilterVersion(packagePURL, packageName, version, upstream.get("time")) { - return nil - } - - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - if routed { - disableDefaultNotification(upstream) - } - h.writeVersion(out, packageName, version, upstream, written) - *kept = append(*kept, packageName+"\x00"+version) - return nil + return composer.Rewrite(body, composer.Options{ + ProxyURL: h.proxyURL, + Cooldown: h.proxy.cooldown(), + EvaluatedAt: time.Now(), + LocalPackage: localPackage, + LocalVersions: local, + Routed: func(packageName string) bool { + _, routed := h.routes.match(packageName) + return routed + }, + OnCooldownFilter: h.cooldownFiltered, }) - if err != nil { - return err - } - for _, e := range local { - pushed := newComposerFields() - if err := pushed.apply(e.entry, false); err != nil { - return err - } - if !minified { - written.reset() - } - if !first { - out.WriteByte(',') - } - first = false - if devReset { - out.WriteString(`"` + composerDevReset + `",`) - devReset = false - } - h.writeVersion(out, packageName, e.version, pushed, written) - *kept = append(*kept, packageName+"\x00"+e.version) - } - out.WriteByte(']') - return nil } -// writeVersion writes the fields of upstream that differ from written, with -// dist pointing at this proxy, and "__unset" for those written has and -// upstream lacks. written is then what the client expands this version to. -func (h *ComposerHandler) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *composerFields) { - out.WriteByte('{') - first := true - writeMember := func(key, value []byte) { - if !first { - out.WriteByte(',') - } - first = false - out.Write(key) - out.WriteByte(':') - out.Write(value) +// cooldownFiltered logs a version a metadata rewrite held back and records +// it for the package page. +func (h *ComposerHandler) cooldownFiltered(packageName, version string, publishedAt time.Time) { + h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) + // A routed package's times are the route's, and aren't stored under the + // default registry's PURLs (see releaseTimeStored). + if _, routed := h.routes.match(packageName); !routed { + h.proxy.recordWithheld("composer", packageName, version, publishedAt) } - - for _, f := range upstream.fields { - value := f.value - if value == nil { - continue - } - if f.name == "dist" { - value = h.rewriteDist(packageName, version, value) - } - if bytes.Equal(value, written.get(f.name)) { - continue - } - writeMember(f.key, value) - written.set(f.name, f.key, value) - } - for _, f := range written.fields { - if f.value != nil && upstream.get(f.name) == nil { - writeMember(f.key, composerUnsetJSON) - written.set(f.name, f.key, nil) - } - } - out.WriteByte('}') } -// shouldFilterVersion returns true if the version should be excluded due to cooldown. +// shouldFilterVersion reports whether cooldown holds a version back from a +// download. func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version string, published []byte) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - - publishedAt, ok := composerPublishTime(published) - if !ok { + publishedAt, ok := composer.PublishTime(published) + if !ok || h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { return false } - - if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { - h.proxy.Logger.Info("cooldown: filtering composer version", - "package", packageName, "version", version) - return true - } - - return false -} - -// composerPublishTime parses the raw "time" field of a version, reporting -// false when it is missing or not an RFC 3339 string. -func composerPublishTime(published []byte) (time.Time, bool) { - var timeStr string - if err := json.Unmarshal(published, &timeStr); err != nil { - return time.Time{}, false - } - publishedAt, err := time.Parse(time.RFC3339, timeStr) - return publishedAt, err == nil -} - -// disableDefaultNotification stops Composer from reporting installs of a routed -// package to the default repository's notify-batch URL, which would disclose -// the package name to it. Composer only falls back to notify-batch when a -// version has no notification-url of its own, and skips an empty one. Setting -// it on the expanded fields keeps it set for the versions that inherit them. -// Composer reads a null notification-url as absent too. -func disableDefaultNotification(fields *composerFields) { - if v := fields.get("notification-url"); v == nil || string(v) == "null" { - fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) - } -} - -// rewriteDist returns a version's dist object with its url pointing at this -// proxy. A dist without a string url is returned unchanged. -func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { - urlMember, ok, err := jsonscan.FindMember(dist, "url") - if err != nil || !ok || dist[urlMember.ValStart] != '"' { - return dist - } - var upstreamURL string - if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { - return dist - } - distType, _, _ := jsonscan.LookupString(dist, "type") - - proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) - if !ok { - return dist - } - encoded, err := json.Marshal(proxyURL) - if err != nil { - return dist - } - rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) - rewritten = append(rewritten, dist[:urlMember.ValStart]...) - rewritten = append(rewritten, encoded...) - return append(rewritten, dist[urlMember.ValEnd:]...) + h.cooldownFiltered(packageName, version, publishedAt) + return true } -// proxyDistURL returns the URL this proxy serves a version's archive at, or -// false when the upstream URL or package name gives nothing to build it from. func (h *ComposerHandler) proxyDistURL(packageName, version, upstreamURL, distType string) (string, bool) { - if upstreamURL == "" { - return "", false - } - parts := strings.SplitN(packageName, "/", vendorPackageParts) - if len(parts) != vendorPackageParts { - return "", false - } - return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", - h.proxyURL, parts[0], parts[1], url.PathEscape(version), distFilename(upstreamURL, distType)), true + return composer.ProxyDistURL(h.proxyURL, packageName, version, upstreamURL, distType) } -// distFilename returns the file name this proxy serves an archive under: the -// last segment of its upstream URL. -func distFilename(upstreamURL, distType string) string { - filename := "package.zip" - if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { - filename = upstreamURL[idx+1:] - } - - // GitHub zipball URLs end with a bare commit hash (no extension). - // Append .zip so the archives library can detect the format. - if path.Ext(filename) == "" && distType == "zip" { - filename += ".zip" - } - return filename -} - -// composerCommitPattern matches a full git commit hash, SHA-1 or SHA-256. -var composerCommitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) - -// upstreamDistURL returns the upstream URL of the archive served as filename, -// given the dist URL the version's metadata lists now. A branch version only -// lists its current head while lock files pin older commits, so for a branch -// a dist URL ending in a commit hash is rebuilt for the requested commit. A -// tagged release has to ask for the listed name. Any other name gives false: -// fetching the listed archive would cache it under a name it does not have. func upstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { - listed := distFilename(listedURL, distType) - // Proxy versions before the .zip suffix handed out bare commit names, - // and lock files written back then still ask for them. - if bare := distFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { - filename += listed[len(bare):] - } - if filename == listed { - return listedURL, true - } - if commit, ok := shaQueryCommit(listed); ok { - requested, ok := shaQueryCommit(filename) - listedSha, requestedSha := "sha="+commit, "sha="+requested - idx := strings.LastIndex(listed, listedSha) - if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { - return "", false - } - idx = strings.LastIndex(listedURL, listedSha) - return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true - } - ext := path.Ext(listed) - commit := strings.TrimSuffix(listed, ext) - requested, ok := strings.CutSuffix(filename, ext) - if !branch || !ok || !composerCommitPattern.MatchString(commit) || !composerCommitPattern.MatchString(requested) { - return "", false - } - idx := strings.LastIndex(listedURL, commit) - return listedURL[:idx] + requested + listedURL[idx+len(commit):], true -} - -// shaQueryCommit returns the commit an archive name passes in a sha query -// parameter. GitLab names the archive of every commit archive.zip and picks -// the commit that way. -func shaQueryCommit(filename string) (string, bool) { - _, query, ok := strings.Cut(filename, "?") - if !ok { - return "", false - } - values, err := url.ParseQuery(query) - if err != nil { - return "", false - } - commit := values.Get("sha") - return commit, composerCommitPattern.MatchString(commit) -} - -// composerFields holds one version's fields as Composer sees them after -// expanding the minified format: raw values under their decoded names, in the -// order the names first appeared. A removed field keeps its slot with a nil -// value, so the order stays stable when it comes back. -type composerFields struct { - fields []composerField - index map[string]int -} - -type composerField struct { - name string - key, value []byte -} - -func newComposerFields() *composerFields { - return &composerFields{index: map[string]int{}} -} - -func (f *composerFields) reset() { - f.fields = f.fields[:0] - clear(f.index) -} - -// get returns the raw value of the named field, or nil when it is not set. -func (f *composerFields) get(name string) []byte { - if i, ok := f.index[name]; ok { - return f.fields[i].value - } - return nil -} - -// string returns the named field when it is a string, or "". -func (f *composerFields) string(name string) string { - raw := f.get(name) - if len(raw) == 0 || raw[0] != '"' { - return "" - } - var s string - _ = json.Unmarshal(raw, &s) - return s -} - -// set stores value under name, written with the quoted key. A nil value -// removes the field. -func (f *composerFields) set(name string, key, value []byte) { - if i, ok := f.index[name]; ok { - f.fields[i].key, f.fields[i].value = key, value - return - } - f.index[name] = len(f.fields) - f.fields = append(f.fields, composerField{name: name, key: key, value: value}) -} - -// apply overlays the members of a version entry. In minified metadata the -// value "__unset" removes the field instead. -func (f *composerFields) apply(entry []byte, minified bool) error { - return jsonscan.EachMember(entry, func(m jsonscan.Member) error { - key, value := m.Key(entry), m.Value(entry) - if minified && jsonscan.StringIs(value, composerUnset) { - value = nil - } - if bytes.IndexByte(key, '\\') < 0 { - if i, ok := f.index[string(key[1:len(key)-1])]; ok { - f.fields[i].key, f.fields[i].value = key, value - return nil - } - } - name, err := jsonscan.Key(key) - if err != nil { - return err - } - f.set(name, key, value) - return nil - }) + return composer.UpstreamDistURL(listedURL, distType, filename, branch) } // handleDownload serves a package file, fetching and caching from upstream if needed. @@ -685,13 +324,13 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) // the name they are cached under. Proxy versions before that cached every // commit under the shared name, which is dropped once a commit is fetched. var sharedFilename string - if commit, ok := shaQueryCommit(requested); ok { + if commit, ok := composer.ShaQueryCommit(requested); ok { sharedFilename = h.routes.cacheFilename(packageName, filename) ext := path.Ext(filename) filename = strings.TrimSuffix(filename, ext) + "-" + commit + ext } cacheFilename := h.routes.cacheFilename(packageName, filename) - if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composerCommitPattern.MatchString(commit) { + if commit := strings.TrimSuffix(filename, path.Ext(filename)); isDevVersion(version) && composer.IsCommit(commit) { h.discardOtherCommit(packageName, version, cacheFilename, commit) } result, err := h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "composer", packageName, version, cacheFilename, downloadURL, @@ -770,7 +409,7 @@ func (h *ComposerHandler) recordPublishTime(packageName, version string, publish if !h.releaseTimeStored(packageName, version) { return } - publishedAt, ok := composerPublishTime(published) + publishedAt, ok := composer.PublishTime(published) if !ok { return } @@ -966,55 +605,10 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU return dist, nil } -// composerDistURL returns the upstream dist of version from Composer -// metadata, with an empty URL when the package has no such version or it has -// no dist URL. It walks the version list in place, expanding only as much of -// the minified format as it needs, since it runs on each archive the proxy -// has not cached yet. +// composerDistURL returns the upstream dist of version; see composer.FindDist. func composerDistURL(body []byte, packageName, version string) (composerDist, error) { - format, _, err := jsonscan.LookupString(body, "minified") - if err != nil { - return composerDist{}, err - } - versions, err := jsonscan.Lookup(body, "packages", packageName) - if err != nil || len(versions) == 0 || versions[0] != '[' { - return composerDist{}, err - } - minified := format == composerMinified - - fields := newComposerFields() - var dist composerDist - err = jsonscan.EachElement(versions, func(entry []byte) error { - if minified && jsonscan.StringIs(entry, composerDevReset) { - fields.reset() - return nil - } - if entry[0] != '{' { - return nil - } - if !minified { - fields.reset() - } - if err := fields.apply(entry, minified); err != nil { - return err - } - if !jsonscan.StringIs(fields.get("version"), version) { - return nil - } - distRaw := fields.get("dist") - dist.url, _, _ = jsonscan.LookupString(distRaw, "url") - if dist.url != "" { - dist.distType, _, _ = jsonscan.LookupString(distRaw, "type") - dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") - dist.published = fields.get("time") - return jsonscan.ErrStop - } - return nil - }) - if errors.Is(err, jsonscan.ErrStop) { - err = nil - } - return dist, err + d, err := composer.FindDist(body, packageName, version) + return composerDist{url: d.URL, distType: d.Type, shasum: d.Shasum, published: d.Published}, err } // composerIntegrity returns the integrity to verify an archive of version @@ -1048,7 +642,7 @@ func ComposerReleaseArchive(version, distURL, distType, shasum string) (filename case strings.ContainsAny(distURL, "?#"): return "", "", fmt.Errorf("the archive URL of %s has a query, which isn't mirrored", version) } - return distFilename(distURL, distType), composerIntegrity(shasum, version, true), nil + return composer.DistFilename(distURL, distType), composerIntegrity(shasum, version, true), nil } // RecordComposerPublishTime stores the publish time of a tagged release from diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index be8a1d6d..0142b344 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -6,8 +6,9 @@ import ( "fmt" "net/http" "net/url" - "strings" "time" + + "github.com/git-pkgs/proxy/internal/metadata/composer" ) // composerRootOnlyKeys are composer.json keys that only apply to the root @@ -15,10 +16,7 @@ import ( var composerRootOnlyKeys = []string{"repositories", "config", "minimum-stability", "prefer-stable"} // composerLocalEntry is a pushed version as Composer metadata lists it. -type composerLocalEntry struct { - version string - entry json.RawMessage -} +type composerLocalEntry = composer.LocalVersion // composerLocalEntries returns the pushed versions of a package for one of // its metadata files: the dev versions for the ~dev file, the tagged ones for @@ -59,7 +57,7 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] if err != nil { return nil, err } - entries = append(entries, composerLocalEntry{version: lv.Version, entry: entry}) + entries = append(entries, composerLocalEntry{Version: lv.Version, Entry: entry}) } return entries, nil } @@ -68,17 +66,6 @@ func (h *ComposerHandler) localDistURL(packageName, version, filename string) st return fmt.Sprintf("%s/composer/files/%s/%s/%s", h.proxyURL, packageName, url.PathEscape(version), filename) } -// pushedComposerVersion reports whether version is one of the pushed ones. -// Tags are listed with and without a leading v, and both name one version. -func pushedComposerVersion(local []composerLocalEntry, version string) bool { - for _, e := range local { - if strings.TrimPrefix(e.version, "v") == strings.TrimPrefix(version, "v") { - return true - } - } - return false -} - // serveMetadataWithLocal serves a package's metadata with its pushed versions // added, bypassing the rewrite cache like the npm handler does. When the // upstream does not know the package, the document holds only the pushed @@ -89,7 +76,7 @@ func (h *ComposerHandler) serveMetadataWithLocal(w http.ResponseWriter, r *http. var err error switch { case errors.Is(upstream.err, ErrUpstreamNotFound): - out, err = localComposerDocument(packageName, local) + out, err = composer.LocalDocument(packageName, local) case upstream.err != nil: if writeUpstreamHeld(w, upstream.err) { return @@ -116,7 +103,7 @@ func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http http.Error(w, "not found", http.StatusNotFound) return } - out, err := localComposerDocument(packageName, local) + out, err := composer.LocalDocument(packageName, local) if err != nil { h.proxy.Logger.Error("failed to build composer metadata from pushed versions", "package", packageName, "error", err) http.Error(w, "failed to build package metadata", http.StatusInternalServerError) @@ -124,11 +111,3 @@ func (h *ComposerHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http } serveRewrittenMetadata(w, r, h.proxy.uncachedRewrite("composer", h.proxyURL, cacheKey, out)) } - -func localComposerDocument(packageName string, local []composerLocalEntry) ([]byte, error) { - entries := make([]json.RawMessage, 0, len(local)) - for _, e := range local { - entries = append(entries, e.entry) - } - return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) -} diff --git a/internal/handler/composer_rewrite_test.go b/internal/handler/composer_rewrite_test.go index 460e38e0..4934389d 100644 --- a/internal/handler/composer_rewrite_test.go +++ b/internal/handler/composer_rewrite_test.go @@ -42,7 +42,7 @@ func composerVersions(t *testing.T, output []byte, packageName string) []map[str } expanded = maps.Clone(expanded) for k, v := range fields { - if v == composerUnset { + if v == "__unset" { delete(expanded, k) continue } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index b765f9a5..fa9120da 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -138,8 +138,8 @@ func (d timedDocuments) rewriteDigest() string { } // withheld reports whether cooldown leaves version out, given the -// package's publish times. -func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version string) bool { +// package's publish times, and records a version it leaves out. +func (d timedDocuments) withheld(p *Proxy, ecosystem, name, version string) bool { published, ok := d.published[version] if !ok { return d.withholdUnknown @@ -147,7 +147,11 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) + if p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), published) { + return false + } + p.recordWithheld(ecosystem, name, version, published) + return true } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/cooldown_withheld.go b/internal/handler/cooldown_withheld.go new file mode 100644 index 00000000..52c12a6a --- /dev/null +++ b/internal/handler/cooldown_withheld.go @@ -0,0 +1,42 @@ +package handler + +import ( + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// recordWithheld stores the publish time of a version a cooldown filter held +// back, so the package page can list it with when it becomes available, +// also for a package nobody has downloaded yet. name is the package name as +// the ecosystem's downloads store it, so both land on the same package row. +// A version without a package name, a version or a known time is skipped, +// and a failed write is only logged: the metadata is served either way. +func (p *Proxy) recordWithheld(ecosystem, name, version string, publishedAt time.Time) { + p.recordWithheldWith(ecosystem, name, version, publishedAt, false) +} + +// recordWithheldPyPI is recordWithheld for a PyPI release, whose time is its +// newest file's: a later time replaces the stored one. +func (p *Proxy) recordWithheldPyPI(name, version string, publishedAt time.Time) { + p.recordWithheldWith("pypi", name, version, publishedAt, true) +} + +func (p *Proxy) recordWithheldWith(ecosystem, name, version string, publishedAt time.Time, newest bool) { + if p.DB == nil || name == "" || version == "" || publishedAt.IsZero() { + return + } + packagePURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return + } + pkg := &database.Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name} + record := p.DB.RecordWithheldVersion + if newest { + record = p.DB.RecordWithheldVersionNewest + } + if err := record(pkg, versionPURL, publishedAt); err != nil { + p.Logger.Warn("cooldown: failed to record withheld version", + "ecosystem", ecosystem, "name", name, "version", version, "error", err) + } +} diff --git a/internal/handler/cooldown_withheld_test.go b/internal/handler/cooldown_withheld_test.go new file mode 100644 index 00000000..55edaab2 --- /dev/null +++ b/internal/handler/cooldown_withheld_test.go @@ -0,0 +1,554 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/database" +) + +// withheldTestProxy returns a proxy with a database and a three day cooldown, +// and the times of a version old enough to pass it and one it holds back. +func withheldTestProxy(t *testing.T) (proxy *Proxy, db *database.DB, old, recent time.Time) { + t.Helper() + proxy, db, _, _ = setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + now := time.Now().UTC().Truncate(time.Second) + return proxy, db, now.Add(-10 * 24 * time.Hour), now.Add(-time.Hour) +} + +// assertWithheldRecorded checks that a version a filter held back is stored +// with its publish time, under a package row the package page finds by +// ecosystem and name. +func assertWithheldRecorded(t *testing.T, db *database.DB, ecosystem, name, version string, published time.Time) { + t.Helper() + pkg, err := db.GetPackageByEcosystemName(ecosystem, name) + if err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil || ver == nil { + t.Fatalf("version %s = %v, %v; want a row", canonicalVersionPURL(ecosystem, name, version), ver, err) + } + if ver.PackagePURL != pkg.PURL { + t.Errorf("version's package = %q, want %q", ver.PackagePURL, pkg.PURL) + } + if !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(published) { + t.Errorf("version %s published %v, want %v", version, ver.PublishedAt, published) + } +} + +// assertNotRecorded checks that a version the filter let through got no row: +// only held back versions are recorded. +func assertNotRecorded(t *testing.T, db *database.DB, ecosystem, name, version string) { + t.Helper() + ver, err := db.GetVersionByPURL(canonicalVersionPURL(ecosystem, name, version)) + if err != nil { + t.Fatalf("GetVersionByPURL: %v", err) + } + if ver != nil { + t.Errorf("version %s recorded although cooldown let it through", version) + } +} + +func TestRecordWithheldNeedsADatabase(t *testing.T) { + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Now()) +} + +func TestRecordWithheldSkipsUnknownTimes(t *testing.T) { + proxy, db, _, _ := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "1.0.0", time.Time{}) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a time", pkg, err) + } +} + +// Without a version the version PURL would be the package's own. +func TestRecordWithheldSkipsVersionsWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("npm", "leftpad", "", recent) + if pkg, err := db.GetPackageByEcosystemName("npm", "leftpad"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a version", pkg, err) + } +} + +func TestNPMMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"testpkg","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"testpkg","version":"1.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-1.0.0.tgz"}}, + "2.0.0":{"name":"testpkg","version":"2.0.0","dist":{"tarball":"https://registry.npmjs.org/testpkg/-/testpkg-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "npm", "testpkg", "2.0.0", recent) + assertNotRecorded(t, db, "npm", "testpkg", "1.0.0") +} + +func TestCargoIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &CargoHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + line := func(version string, at time.Time) string { + return `{"name":"serde","vers":"` + version + `","cksum":"abcd","features":{},"yanked":false,"pubtime":"` + at.Format(time.RFC3339) + `"}` + "\n" + } + h.applyCooldownFiltering(httptest.NewRecorder(), []byte(line("1.0.0", old)+line("1.0.1", recent))) + assertWithheldRecorded(t, db, "cargo", "serde", "1.0.1", recent) + assertNotRecorded(t, db, "cargo", "serde", "1.0.0") +} + +func TestComposerMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"packages":{"symfony/console":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://repo.packagist.org/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertWithheldRecorded(t, db, "composer", "symfony/console", "6.0.0", recent) + assertNotRecorded(t, db, "composer", "symfony/console", "5.0.0") +} + +func TestPubMetadataRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + input := `{"name":"flutter_bloc","latest":{"version":"2.0.0"},"versions":[ + {"version":"1.0.0","published":"` + old.Format(time.RFC3339) + `","archive_url":"https://pub.dev/1.0.0.tar.gz"}, + {"version":"2.0.0","published":"` + recent.Format(time.RFC3339) + `","archive_url":"https://pub.dev/2.0.0.tar.gz"}]}` + if _, err := h.rewriteMetadata("flutter_bloc", []byte(input)); err != nil { + t.Fatalf("rewriteMetadata: %v", err) + } + assertWithheldRecorded(t, db, "pub", "flutter_bloc", "2.0.0", recent) + assertNotRecorded(t, db, "pub", "flutter_bloc", "1.0.0") +} + +func TestTimedDocumentsRecordWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + docs := timedDocuments{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent, "3.0.0": {}}} + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0", "4.0.0"} { + docs.withheld(proxy, "hex", "testpkg", v) + } + assertWithheldRecorded(t, db, "hex", "testpkg", "2.0.0", recent) + for _, v := range []string{"1.0.0", "3.0.0", "4.0.0"} { + assertNotRecorded(t, db, "hex", "testpkg", v) + } +} + +func TestGemIndexRecordsWithheldVersions(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + versionsJSON, _ := json.Marshal([]gemVersion{ + {Number: "1.0.0", Platform: "ruby", CreatedAt: old.Format(time.RFC3339)}, + {Number: "2.0.0", Platform: "ruby", CreatedAt: recent.Format(time.RFC3339)}, + }) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasPrefix(r.URL.Path, "/info/"): + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("---\n1.0.0 dep1:>= 1.0|checksum:abc123\n2.0.0 dep1:>= 1.0|checksum:def456\n")) + case strings.HasPrefix(r.URL.Path, "/api/v1/versions/"): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(versionsJSON) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + h := &GemHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://proxy.local"} + + req := httptest.NewRequest(http.MethodGet, "/info/testgem", nil) + req.SetPathValue("name", "testgem") + w := httptest.NewRecorder() + h.handleCompactIndex(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) + } + assertWithheldRecorded(t, db, "gem", "testgem", "2.0.0", recent) + assertNotRecorded(t, db, "gem", "testgem", "1.0.0") +} + +// Go stores the publish time of every version it reads, so what the filter +// adds is the package row. +func TestGoListRecordsWithheldVersions(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + +// A .info request for a version in cooldown is refused, and the version is +// recorded with the publish time the refusal was based on. +func TestGoRefusedInfoRecordsWithheldVersion(t *testing.T) { + u := newGoCooldownUpstream(t, "v1.1.0") + h, _, proxy := goCooldownHandler(t, u) + if w := goGet(h, "/example.com/mod/@v/v1.1.0.info"); w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want 403", w.Code) + } + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/mod", "v1.1.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("version = %v, %v; want it stored with its publish time", ver, err) + } + assertPackageRow(t, proxy.DB, "golang", "example.com/mod") +} + +func assertPackageRow(t *testing.T, db *database.DB, ecosystem, name string) { + t.Helper() + if pkg, err := db.GetPackageByEcosystemName(ecosystem, name); err != nil || pkg == nil { + t.Fatalf("package %s/%s = %v, %v; want a row", ecosystem, name, pkg, err) + } +} + +func TestMavenMetadataRecordsWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + for _, v := range []string{"2.0", "2.1"} { + assertWithheldRecorded(t, proxy.DB, "maven", "com.example:lib", v, u.times[v].UTC().Truncate(time.Second)) + } +} + +func TestMavenNamedRepositoryRecordsNoWithheldVersions(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + if w := getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if pkg, err := db.GetPackageByEcosystemName("maven", "com.example:lib"); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want no row from a named repository", pkg, err) + } +} + +func TestPyPISimpleRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + body := `{"meta":{"api-version":"1.4"},"name":"ruff","files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload-time":"` + old.Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload-time":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, pypiSimpleJSON, body), nil + case "/pypi/ruff/json": + body := `{"releases":{"1.0.0":[{"upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + default: + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + return nil, nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + req := httptest.NewRequest(http.MethodGet, "/simple/ruff/", nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// The JSON API filters releases on its own path, so a client that reads it +// rather than the simple index gets the held back version recorded too. +func TestPyPIJSONRecordsWithheldVersions(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + old, recent := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/pypi/ruff/json" { + t.Fatalf("unexpected upstream path: %s", r.URL.Path) + } + body := `{"info":{"name":"ruff","version":"1.0.0"},"releases":{` + + `"1.0.0":[{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz","upload_time_iso_8601":"` + old.Format(time.RFC3339) + `"}],` + + `"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Add(-time.Minute).Format(time.RFC3339) + `"},` + + `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), `"2.0.0"`) { + t.Fatalf("2.0.0 was served although it is in cooldown") + } + // A release's time is its newest file's. + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) + assertNotRecorded(t, proxy.DB, "pypi", "ruff", "1.0.0") +} + +// A release's time is its newest file's, and files can be added to a release +// after it was first held back, so the newer time replaces the stored one. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + files := []string{`{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + first.Format(time.RFC3339) + `"}`} + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"releases":{"2.0.0":[` + strings.Join(files, ",") + `]}}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + files = append(files, `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl","upload_time_iso_8601":"`+added.Format(time.RFC3339)+`"}`) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) +} + +// The simple index and the version JSON also keep the newest upload time. +func TestPyPIWithheldReleaseKeepsItsNewestUploadTimeOnEveryPath(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + first, added := now.Add(-2*time.Hour), now.Add(-time.Hour) + for name, path := range map[string]string{"simple": "/simple/ruff/", "version json": "/pypi/ruff/2.0.0/json"} { + t.Run(name, func(t *testing.T) { + times := []time.Time{first} + files := func(key string) string { + var out []string + for i, at := range times { + out = append(out, fmt.Sprintf(`{"filename":"ruff-2.0.0-%d.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0-%d.tar.gz",%q:%q}`, i, i, key, at.Format(time.RFC3339))) + } + return strings.Join(out, ",") + } + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + switch r.URL.Path { + case "/simple/ruff/": + return pypiHTTPResponse(r, pypiSimpleJSON, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[`+files("upload-time")+`]}`), nil + case "/pypi/ruff/json": + return pypiHTTPResponse(r, "application/json", `{"releases":{"2.0.0":[`+files("upload_time_iso_8601")+`]}}`), nil + default: + return pypiHTTPResponse(r, "application/json", `{"info":{"name":"ruff","version":"2.0.0"},"urls":[`+files("upload_time_iso_8601")+`]}`), nil + } + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + get := func() { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Accept", uvPyPIAccept) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + } + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", first) + times = append(times, added) + get() + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", added) + }) + } +} + +// A download that read an older document than one already recorded leaves +// the newer upload time, so cooldown doesn't end early. +func TestPyPIDownloadKeepsTheNewerStoredUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + older, newer := now.Add(-2*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", + `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.recordWithheldPyPI("ruff", "2.0.0", newer) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown { + t.Fatalf("versionInCooldown = %v, %v; want true", inCooldown, err) + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", newer) +} + +// A download that read an upload time out of cooldown is still refused when +// a newer time, inside it, was stored meanwhile. +func TestPyPIDownloadDecidesWithTheNewerStoredUploadTime(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + older, newer := now.Add(-30*24*time.Hour), now.Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, "application/json", + `{"releases":{"2.0.0":[{"filename":"ruff-2.0.0.tar.gz","upload_time_iso_8601":"`+older.Format(time.RFC3339)+`"}]}}`), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.recordWithheldPyPI("ruff", "2.0.0", newer) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + if inCooldown, err := h.versionInCooldown(req, "ruff", "2.0.0"); err != nil || !inCooldown { + t.Errorf("versionInCooldown = %v, %v; want true from the newer stored time", inCooldown, err) + } +} + +// The version JSON names one release, in urls, and records it when it is +// held back. +func TestPyPIVersionJSONRecordsWithheldVersion(t *testing.T) { + recent := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + h, proxy := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + body := `{"info":{"name":"ruff","version":"2.0.0"},"urls":[` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz","upload_time_iso_8601":"` + recent.Format(time.RFC3339) + `"}]}` + return pypiHTTPResponse(r, "application/json", body), nil + }) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/ruff/2.0.0/json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if strings.Contains(w.Body.String(), "ruff-2.0.0.tar.gz") { + t.Fatalf("the files of 2.0.0 were served although it is in cooldown") + } + assertWithheldRecorded(t, proxy.DB, "pypi", "ruff", "2.0.0", recent) +} + +// The NuGet listing and the registration each find the held back version. +func TestNuGetMetadataRecordsWithheldVersions(t *testing.T) { + for _, path := range []string{ + "/nuget/v3-flatcontainer/testpkg/index.json", + "/nuget" + nugetRegistrationPath + "testpkg/index.json", + } { + t.Run(path, func(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), path, http.StatusOK) + + assertPackageRow(t, db, "nuget", "testpkg") + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } + if age := time.Since(ver.PublishedAt.Time); age < 47*time.Hour || age > 49*time.Hour { + t.Errorf("2.0.0 published %v ago, want two days", age) + } + assertNotRecorded(t, db, "nuget", "testpkg", "1.0.0") + }) + } +} + +// Without a name the PURLs would name no package, such as pkg:cargo/. +func TestRecordWithheldSkipsPackagesWithoutAName(t *testing.T) { + proxy, db, _, recent := withheldTestProxy(t) + proxy.recordWithheld("cargo", "", "1.0.0", recent) + if pkg, err := db.GetPackageByEcosystemName("cargo", ""); err != nil || pkg != nil { + t.Errorf("package = %v, %v; want nothing recorded without a name", pkg, err) + } +} + +// A routed package's times come from its route and are never stored under +// the default registry's PURLs, so the filter doesn't record them either. +func TestNPMMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &NPMHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"@corp/*": "https://npm.corp.example"})} + input := `{"name":"@corp/lib","dist-tags":{"latest":"2.0.0"}, + "time":{"1.0.0":"` + old.Format(time.RFC3339) + `","2.0.0":"` + recent.Format(time.RFC3339) + `"}, + "versions":{ + "1.0.0":{"name":"@corp/lib","version":"1.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-1.0.0.tgz"}}, + "2.0.0":{"name":"@corp/lib","version":"2.0.0","dist":{"tarball":"https://npm.corp.example/@corp/lib/-/lib-2.0.0.tgz"}}}}` + if _, _, err := h.rewriteMetadataKeeping("@corp/lib", []byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "npm", "@corp/lib", "2.0.0") +} + +func TestComposerMetadataOfRoutedPackageRecordsNothing(t *testing.T) { + proxy, db, old, recent := withheldTestProxy(t) + h := &ComposerHandler{proxy: proxy, proxyURL: "http://localhost:8080", + routes: newPackageRoutes(map[string]string{"corp/*": "https://composer.corp.example"})} + input := `{"packages":{"corp/lib":[ + {"version":"5.0.0","time":"` + old.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/5.0.0.zip","type":"zip"}}, + {"version":"6.0.0","time":"` + recent.Format(time.RFC3339) + `","dist":{"url":"https://composer.corp.example/6.0.0.zip","type":"zip"}}]}}` + if _, _, err := h.rewriteMetadataKeeping([]byte(input)); err != nil { + t.Fatalf("rewriteMetadataKeeping: %v", err) + } + assertNotRecorded(t, db, "composer", "corp/lib", "6.0.0") +} + +// Go keeps publish times under the escaped module path, which differs from +// the module path downloads are stored under when it has capitals. Recording +// such a module would give one release two version rows, so it is skipped. +func TestGoListOfModuleWithCapitalsRecordsNoSecondRow(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/example.com/!mod/@v/list": + _, _ = w.Write([]byte("v1.1.0\n")) + case "/example.com/!mod/@v/v1.1.0.info": + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.1.0", Time: now.Add(-time.Hour)}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewGoHandlerWithUpstream(proxy, "http://localhost:8080", upstream.URL).Routes() + if w := goGet(h, "/example.com/!mod/@v/list"); w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + if ver, err := db.GetVersionByPURL(canonicalVersionPURL("golang", "example.com/!mod", "v1.1.0")); err != nil || ver == nil { + t.Fatalf("time under the escaped path = %v, %v; want it stored", ver, err) + } + assertNotRecorded(t, db, "golang", "example.com/Mod", "v1.1.0") +} + +// A registration leaf carries no version of its own; it is the one its path +// names. +func TestNuGetRegistrationLeafRecordsWithheldVersion(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, http.StripPrefix("/nuget", h.Routes()), "/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound) + + ver, err := db.GetVersionByPURL(canonicalVersionPURL("nuget", "testpkg", "2.0.0")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("2.0.0 = %+v, %v; want a row with its publish time", ver, err) + } +} + +func TestNuGetRegistrationLeafVersion(t *testing.T) { + for path, want := range map[string]string{ + nugetRegistrationPath + "testpkg/2.0.0.json": "2.0.0", + nugetRegistrationPath + "testpkg/index.json": "", + nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": "", + nugetRegistrationPath + "testpkg/2.0.0": "", + "/other/testpkg/2.0.0.json": "", + } { + if got := nugetRegistrationLeafVersion(path); got != want { + t.Errorf("nugetRegistrationLeafVersion(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/internal/handler/gem.go b/internal/handler/gem.go index fa7c24ba..25983cdc 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -176,7 +176,6 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return } - packagePURL := canonicalPackagePURL("gem", name) m, err := h.proxy.cachedRewrite(r.Context(), "gem", h.proxyURL, "info/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -184,7 +183,7 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) return nil, nil, err } out, kept := h.filterIndex(body, name, func(version string) bool { - return docs.withheld(h.proxy, "gem", packagePURL, version) + return docs.withheld(h.proxy, "gem", name, version) }) return out, kept, nil }) diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index efcc1e19..e788e688 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -168,6 +168,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s } keep := make([]bool, len(versions)) + times := make([]time.Time, len(versions)) g, gctx := errgroup.WithContext(ctx) g.SetLimit(goInfoFetchLimit) for i, v := range versions { @@ -177,6 +178,7 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s return fmt.Errorf("reading time of %s: %w", v, err) } keep[i] = h.allowedAt(module, published) + times[i] = published return nil }) } @@ -190,11 +192,23 @@ func (h *GoHandler) allowedVersions(ctx context.Context, escapedModule, module s allowed = append(allowed, v) } else { h.proxy.Logger.Info("cooldown: filtering go module version", "module", module, "version", v) + h.recordWithheld(module, v, times[i]) } } return allowed, nil } +// recordWithheld records a version the list filter held back. Publish times +// are stored under the escaped module path and downloads under the module +// path, so a module whose path has capitals, and the two differ, is skipped +// rather than given a second row for the same release. +func (h *GoHandler) recordWithheld(module, version string, published time.Time) { + if escaped, err := gomodule.EscapePath(module); err != nil || escaped != module { + return + } + h.proxy.recordWithheld("golang", module, version, published) +} + // serveCooldownList serves @v/list without versions in cooldown. func (h *GoHandler) serveCooldownList(w http.ResponseWriter, r *http.Request, escapedModule, module string) { allowed, err := h.allowedVersions(r.Context(), escapedModule, module) @@ -220,6 +234,7 @@ func (h *GoHandler) serveCooldownInfo(w http.ResponseWriter, r *http.Request, es var info goInfo if err := json.Unmarshal(body, &info); err == nil && info.Version != "" { if !h.allowedAt(module, info.Time) { + h.recordWithheld(module, info.Version, info.Time) h.refuseCooldown(w, module, info.Version) return } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index d394c4df..17899f10 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -130,7 +130,6 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { return } - packagePURL := canonicalPackagePURL("hex", name) m, err := h.proxy.cachedRewrite(r.Context(), "hex", h.proxyURL, "packages/"+name, docs.doc.body, docs.rewriteDigest(), func([]byte) ([]byte, []string, error) { body, err := h.proxy.decodedMetadataBody(docs.doc) @@ -144,7 +143,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { filtered := make(map[string]bool) var kept []string for _, version := range versions { - if docs.withheld(h.proxy, "hex", packagePURL, version) { + if docs.withheld(h.proxy, "hex", name, version) { filtered[version] = true h.proxy.Logger.Info("cooldown: filtering hex version", "package", name, "version", version) continue diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 6155e0d7..beccb703 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -330,6 +330,11 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) removed[v] = true + // A named repository's times are its own, under PURLs that name it, + // and the package page shows the default repository's. + if source.name == "" { + h.proxy.recordWithheld("maven", group+":"+artifact, v, published) + } } if len(removed) == 0 { return body, nil diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 3cf3244d..95308b78 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -721,6 +721,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l // cooldown is evaluated at, and the pushed versions to list. The callbacks // log what the rewrite did. func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { + _, routed := h.routes.match(packageName) return npm.Options{ ProxyURL: h.proxyURL, Cooldown: h.proxy.cooldown(), @@ -736,6 +737,11 @@ func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry OnCooldownFilter: func(version, publishedAt string) { h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, "published", publishedAt) + // A routed package's times are the route's, and aren't stored + // under the default registry's PURLs (see versionInCooldown). + if at, err := time.Parse(time.RFC3339, publishedAt); err == nil && !routed { + h.proxy.recordWithheld("npm", packageName, version, at) + } }, } } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 714f2192..1e6615a5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -393,21 +393,27 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str return false, err } _, published := leaf.release() - return h.nugetPublishedAllowed(id, published), nil + return h.nugetPublishedAllowed(id, version, published), nil } // Missing/invalid timestamps retain the existing permissive behavior, but -// fetch and JSON errors must not bypass the policy. -func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { +// fetch and JSON errors must not bypass the policy. A version held back is +// recorded under the lowercase ID and version downloads are stored under. +func (h *NuGetHandler) nugetPublishedAllowed(id, version, published string) bool { when, err := time.Parse(time.RFC3339, published) if err != nil { return true } - return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + name := strings.ToLower(id) + if h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", name), when) { + return true + } + h.proxy.recordWithheld("nuget", name, nugetVersionKey(version), when) + return false } func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) { - if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) { + if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, version, published) { blocked[nugetVersionKey(version)] = true } for i := range document.Items { @@ -436,7 +442,11 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request return } if applyCooldown && document.Items == nil { - if _, published := document.release(); !h.nugetPublishedAllowed(id, published) { + version, published := document.release() + if version == "" { + version = nugetRegistrationLeafVersion(r.URL.Path) + } + if !h.nugetPublishedAllowed(id, version, published) { JSONError(w, http.StatusNotFound, "version not found") return } @@ -469,6 +479,25 @@ func nugetRegistrationID(path string) string { return "" } +// nugetRegistrationLeafVersion returns the version a registration leaf's +// path names ({prefix}{id}/{version}.json), or "" for any other path. A leaf +// as nuget.org serves it carries no version of its own. +func nugetRegistrationLeafVersion(path string) string { + for _, prefix := range nugetRegistrationPrefixes { + rest, ok := strings.CutPrefix(path, prefix) + if !ok { + continue + } + _, leaf, _ := strings.Cut(rest, "/") + version, isJSON := strings.CutSuffix(leaf, ".json") + if !isJSON || version == "" || version == "index" || strings.Contains(version, "/") { + return "" + } + return version + } + return "" +} + // expandNuGetPageTimes gives each page of a registration index that only // links to its leaves the leaves of the page it links to. With bodies not nil // the linked pages themselves are kept in it, by link. Only index pages are diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go index 01da55de..e32273ff 100644 --- a/internal/handler/nuget_registration.go +++ b/internal/handler/nuget_registration.go @@ -389,7 +389,11 @@ func (rw *nugetRegistrationWriter) node(dec *jsontext.Decoder, depth int) (bool, if n.container { return n.kept > 0, release.version, nil } - return rw.h.nugetPublishedAllowed(rw.id, release.published), release.version, nil + var version string + if len(release.version) >= 2 && release.version[0] == '"' { + version = string(rw.unquote(release.version)) + } + return rw.h.nugetPublishedAllowed(rw.id, version, release.published), release.version, nil } // linksPage reports whether a node at depth is a page of an index whose diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 5a6d426c..1fe8f040 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -288,6 +288,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) + h.proxy.recordWithheld("pub", name, version, publishedAt) return true } diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index c94e1515..3580140a 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -326,6 +326,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s for version, publishedAt := range times { if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true + h.proxy.recordWithheldPyPI(name, version, publishedAt) } } @@ -625,9 +626,17 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } for version, files := range releases { - if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) { + denied := h.proxy.versionDenied("pypi", packageName, version) + held, publishedAt := false, time.Time{} + if !denied { + held, publishedAt = h.shouldFilterRelease(packagePURL, files) + } + if denied || held { h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) + if held { + h.proxy.recordWithheldPyPI(packageName, version, publishedAt) + } delete(releases, version) continue } @@ -636,19 +645,20 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } } -// shouldFilterRelease returns true if a release should be excluded due to cooldown. -func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { +// shouldFilterRelease reports whether a release should be excluded due to +// cooldown, and the newest upload time that decided it. +func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) (bool, time.Time) { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() || packagePURL == "" { - return false + return false, time.Time{} } filesArr, ok := files.([]any) if !ok { - return false + return false, time.Time{} } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt), publishedAt } // versionInCooldown reports whether a version is still inside the cooldown @@ -658,7 +668,8 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { // // A release whose upload time cannot be determined is allowed through, matching // how fetchFilteredVersions treats it. Failing to read the times is an error. -// The time read is stored for storedCooldown. +// The time read is stored for storedCooldown, and a newer one stored +// meanwhile decides. func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) (bool, error) { if !h.proxy.cooldownApplies("pypi", name) { return false, nil @@ -672,24 +683,32 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( if publishedAt.IsZero() { return false, nil } - h.storeUploadTime(name, version, publishedAt) + publishedAt = h.storeUploadTime(name, version, publishedAt) return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } // storeUploadTime records a release's newest upload time under the version -// PURL the file is cached under, so storedCooldown finds it. -func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) { +// PURL the file is cached under, so storedCooldown finds it. The time only +// moves forward: a request that read an older document leaves a newer one, +// and gets that newer time back. +func (h *PyPIHandler) storeUploadTime(name, version string, publishedAt time.Time) time.Time { if h.proxy.DB == nil { - return + return publishedAt } packagePURL, versionPURL, err := packagePURLStrings("pypi", name, version) if err != nil { - return + return publishedAt } - if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { + if err := h.proxy.DB.AdvanceVersionPublishedAt(versionPURL, packagePURL, publishedAt); err != nil { h.proxy.Logger.Warn("cooldown: could not store pypi upload time", "package", name, "version", version, "error", err) + return publishedAt + } + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && + ver.PublishedAt.Valid && ver.PublishedAt.Time.After(publishedAt) { + return ver.PublishedAt.Time } + return publishedAt } // storedCooldown answers the cooldown check for a download from the upload @@ -744,7 +763,10 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL info, _ := metadata["info"].(map[string]any) name, _ := info["name"].(string) version, _ := info["version"].(string) - if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) { + if h.proxy.versionDenied("pypi", name, version) { + metadata["urls"] = []any{} + } else if held, publishedAt := h.shouldFilterRelease(packagePURL, urls); held { + h.proxy.recordWithheldPyPI(name, version, publishedAt) metadata["urls"] = []any{} } diff --git a/internal/metadata/composer/imports_test.go b/internal/metadata/composer/imports_test.go new file mode 100644 index 00000000..0a10bb5c --- /dev/null +++ b/internal/metadata/composer/imports_test.go @@ -0,0 +1,44 @@ +package composer + +import ( + "os/exec" + "slices" + "strings" + "testing" +) + +// TestPortableImports keeps the package free of the proxy's infrastructure: +// no handler, database, storage, metrics, HTTP or logging, so it builds for +// targets such as WASI where those are not available. +func TestPortableImports(t *testing.T) { + goTool, err := exec.LookPath("go") + if err != nil { + t.Skip("go tool not found") + } + out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + allowed := []string{ + "github.com/git-pkgs/cooldown", + "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/metadata/composer", + "github.com/git-pkgs/proxy/internal/packageurl", + "github.com/git-pkgs/purl", + "github.com/git-pkgs/vers", + "github.com/package-url/packageurl-go", + } + forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"} + for line := range strings.Lines(string(out)) { + standard, path, _ := strings.Cut(strings.TrimSpace(line), " ") + if standard == "true" { + if slices.Contains(forbiddenStd, path) { + t.Errorf("imports %s", path) + } + continue + } + if !slices.Contains(allowed, path) { + t.Errorf("imports %s, which is not on the portable list", path) + } + } +} diff --git a/internal/metadata/composer/metadata.go b/internal/metadata/composer/metadata.go new file mode 100644 index 00000000..55e35fc6 --- /dev/null +++ b/internal/metadata/composer/metadata.go @@ -0,0 +1,545 @@ +// Package composer transforms Composer repository metadata without storage or +// network access. +package composer + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/url" + "path" + "regexp" + "strings" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +const ( + // Minified is the "minified" value of Composer 2 metadata, whose + // version lists only carry the fields that change. + Minified = "composer/2.0" + // DevReset starts over the expanded fields in a minified list, before + // the dev versions. + DevReset = "~dev" + unset = "__unset" + vendorPackageParts = 2 +) + +var unsetJSON = []byte(`"` + unset + `"`) + +// CooldownPolicy evaluates version age at the caller's supplied time. +type CooldownPolicy interface { + Enabled() bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision +} + +// LocalVersion is a pushed version as Composer metadata lists it. +type LocalVersion struct { + Version string + Entry json.RawMessage +} + +// Options supplies the public URL, policy and pushed versions for a rewrite. +// EvaluatedAt is used as given; the transformation never reads the clock. +type Options struct { + ProxyURL string + Cooldown CooldownPolicy + EvaluatedAt time.Time + // LocalPackage and LocalVersions are the pushed versions to list. + LocalPackage string + LocalVersions []LocalVersion + // Routed reports whether a package is served from a route rather than + // the default repository; its versions don't report installs to the + // default repository. + Routed func(packageName string) bool + // OnCooldownFilter is called synchronously for each version cooldown + // holds back, with its publish time. + OnCooldownFilter func(packageName, version string, publishedAt time.Time) +} + +// Dist is the upstream archive of a version: its URL, type and shasum, and +// the raw publish time the cooldown is checked against. +type Dist struct { + URL, Type, Shasum string + Published []byte +} + +// Rewrite filters the versions cooldown holds back and points each dist at +// the proxy, returning the kept package names and versions without expanding +// minified lists. The pushed versions in o.LocalVersions are listed under +// o.LocalPackage in place of any upstream entry with the same version; +// cooldown only filters upstream versions. A document that does not list +// o.LocalPackage at all is replaced by one holding only the pushed versions. +func Rewrite(body []byte, o Options) ([]byte, []string, error) { + localPackage, local := o.LocalPackage, o.LocalVersions + if !json.Valid(body) { + return nil, nil, errors.New("composer metadata is not valid JSON") + } + format, _, err := jsonscan.LookupString(body, "minified") + if err != nil { + return nil, nil, err + } + minified := format == Minified + + var out bytes.Buffer + var kept []string + listed := false + out.Grow(len(body) + len(body)/8) + err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { + packages := m.Value(body) + if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { + return false, nil + } + return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { + versions := p.Value(packages) + if versions[0] != '[' { + return false, nil + } + packageName, err := jsonscan.Key(p.Key(packages)) + if err != nil { + return false, err + } + var pushed []LocalVersion + if packageName == localPackage { + pushed, listed = local, true + } + return true, o.writeVersions(out, packageName, versions, minified, &kept, pushed) + }) + }) + if err != nil { + return nil, nil, err + } + if len(local) > 0 && !listed { + document, err := LocalDocument(localPackage, local) + return document, nil, err + } + return out.Bytes(), kept, nil +} + +// writeVersions writes one package's version list without the versions in +// cooldown and with each dist pointing at this proxy, and appends each version +// it writes to kept as its package name and version. +// +// Minified lists stay minified. Each version written carries the fields that +// differ from what the client has expanded so far, which is the upstream +// entry itself unless filtered versions were dropped before it; then it also +// carries the changes those versions made. Every version gets its own dist, +// since the proxied URL names the version. +// +// The pushed versions in local follow the upstream ones and take the place of +// upstream entries with the same version. +func (o Options) writeVersions(out *bytes.Buffer, packageName string, versions []byte, minified bool, kept *[]string, local []LocalVersion) error { + packagePURL := packageurl.MakeString("composer", packageName, "") + upstream, written := newFields(), newFields() + devReset, first := false, true + routed := o.Routed != nil && o.Routed(packageName) + + out.WriteByte('[') + err := jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, DevReset) { + upstream.reset() + written.reset() + devReset = true + return nil + } + if entry[0] != '{' { + return nil + } + if !minified { + upstream.reset() + written.reset() + } + if err := upstream.apply(entry, minified); err != nil { + return err + } + + version := upstream.string("version") + if PushedVersion(local, version) || o.heldBack(packagePURL, packageName, version, upstream.get("time")) { + return nil + } + + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + DevReset + `",`) + devReset = false + } + if routed { + disableDefaultNotification(upstream) + } + o.writeVersion(out, packageName, version, upstream, written) + *kept = append(*kept, packageName+"\x00"+version) + return nil + }) + if err != nil { + return err + } + for _, e := range local { + pushed := newFields() + if err := pushed.apply(e.Entry, false); err != nil { + return err + } + if !minified { + written.reset() + } + if !first { + out.WriteByte(',') + } + first = false + if devReset { + out.WriteString(`"` + DevReset + `",`) + devReset = false + } + o.writeVersion(out, packageName, e.Version, pushed, written) + *kept = append(*kept, packageName+"\x00"+e.Version) + } + out.WriteByte(']') + return nil +} + +// writeVersion writes the fields of upstream that differ from written, with +// dist pointing at this proxy, and "__unset" for those written has and +// upstream lacks. written is then what the client expands this version to. +func (o Options) writeVersion(out *bytes.Buffer, packageName, version string, upstream, written *fields) { + out.WriteByte('{') + first := true + writeMember := func(key, value []byte) { + if !first { + out.WriteByte(',') + } + first = false + out.Write(key) + out.WriteByte(':') + out.Write(value) + } + + for _, f := range upstream.fields { + value := f.value + if value == nil { + continue + } + if f.name == "dist" { + value = o.rewriteDist(packageName, version, value) + } + if bytes.Equal(value, written.get(f.name)) { + continue + } + writeMember(f.key, value) + written.set(f.name, f.key, value) + } + for _, f := range written.fields { + if f.value != nil && upstream.get(f.name) == nil { + writeMember(f.key, unsetJSON) + written.set(f.name, f.key, nil) + } + } + out.WriteByte('}') +} + +// heldBack reports whether cooldown holds a version back. +func (o Options) heldBack(packagePURL, packageName, version string, published []byte) bool { + if o.Cooldown == nil || !o.Cooldown.Enabled() { + return false + } + publishedAt, ok := PublishTime(published) + if !ok || o.Cooldown.Evaluate("composer", packagePURL, publishedAt, o.EvaluatedAt).Allowed { + return false + } + if o.OnCooldownFilter != nil { + o.OnCooldownFilter(packageName, version, publishedAt) + } + return true +} + +// PublishTime parses the raw "time" field of a version, reporting false when +// it is missing or not an RFC 3339 string. +func PublishTime(published []byte) (time.Time, bool) { + var timeStr string + if err := json.Unmarshal(published, &timeStr); err != nil { + return time.Time{}, false + } + publishedAt, err := time.Parse(time.RFC3339, timeStr) + return publishedAt, err == nil +} + +// disableDefaultNotification stops Composer from reporting installs of a routed +// package to the default repository's notify-batch URL, which would disclose +// the package name to it. Composer only falls back to notify-batch when a +// version has no notification-url of its own, and skips an empty one. Setting +// it on the expanded fields keeps it set for the versions that inherit them. +// Composer reads a null notification-url as absent too. +func disableDefaultNotification(fields *fields) { + if v := fields.get("notification-url"); v == nil || string(v) == "null" { + fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) + } +} + +// rewriteDist returns a version's dist object with its url pointing at this +// proxy. A dist without a string url is returned unchanged. +func (o Options) rewriteDist(packageName, version string, dist []byte) []byte { + urlMember, ok, err := jsonscan.FindMember(dist, "url") + if err != nil || !ok || dist[urlMember.ValStart] != '"' { + return dist + } + var upstreamURL string + if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { + return dist + } + distType, _, _ := jsonscan.LookupString(dist, "type") + + proxyURL, ok := ProxyDistURL(o.ProxyURL, packageName, version, upstreamURL, distType) + if !ok { + return dist + } + encoded, err := json.Marshal(proxyURL) + if err != nil { + return dist + } + rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) + rewritten = append(rewritten, dist[:urlMember.ValStart]...) + rewritten = append(rewritten, encoded...) + return append(rewritten, dist[urlMember.ValEnd:]...) +} + +// ProxyDistURL returns the URL the proxy at proxyURL serves a version's +// archive at, or false when the upstream URL or package name gives nothing to +// build it from. +func ProxyDistURL(proxyURL, packageName, version, upstreamURL, distType string) (string, bool) { + if upstreamURL == "" { + return "", false + } + parts := strings.SplitN(packageName, "/", vendorPackageParts) + if len(parts) != vendorPackageParts { + return "", false + } + return fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", + proxyURL, parts[0], parts[1], url.PathEscape(version), DistFilename(upstreamURL, distType)), true +} + +// DistFilename returns the file name the proxy serves an archive under: the +// last segment of its upstream URL. +func DistFilename(upstreamURL, distType string) string { + filename := "package.zip" + if idx := strings.LastIndex(upstreamURL, "/"); idx >= 0 { + filename = upstreamURL[idx+1:] + } + + // GitHub zipball URLs end with a bare commit hash (no extension). + // Append .zip so the archives library can detect the format. + if path.Ext(filename) == "" && distType == "zip" { + filename += ".zip" + } + return filename +} + +// commitPattern matches a full git commit hash, SHA-1 or SHA-256. +var commitPattern = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`) + +// IsCommit reports whether s is a full git commit hash, SHA-1 or SHA-256. +func IsCommit(s string) bool { + return commitPattern.MatchString(s) +} + +// UpstreamDistURL returns the upstream URL of the archive served as filename, +// given the dist URL the version's metadata lists now. A branch version only +// lists its current head while lock files pin older commits, so for a branch +// a dist URL ending in a commit hash is rebuilt for the requested commit. A +// tagged release has to ask for the listed name. Any other name gives false: +// fetching the listed archive would cache it under a name it does not have. +func UpstreamDistURL(listedURL, distType, filename string, branch bool) (string, bool) { + listed := DistFilename(listedURL, distType) + // Proxy versions before the .zip suffix handed out bare commit names, + // and lock files written back then still ask for them. + if bare := DistFilename(listedURL, ""); listed != bare && path.Ext(filename) == "" { + filename += listed[len(bare):] + } + if filename == listed { + return listedURL, true + } + if commit, ok := ShaQueryCommit(listed); ok { + requested, ok := ShaQueryCommit(filename) + listedSha, requestedSha := "sha="+commit, "sha="+requested + idx := strings.LastIndex(listed, listedSha) + if !branch || !ok || idx < 0 || listed[:idx]+requestedSha+listed[idx+len(listedSha):] != filename { + return "", false + } + idx = strings.LastIndex(listedURL, listedSha) + return listedURL[:idx] + requestedSha + listedURL[idx+len(listedSha):], true + } + ext := path.Ext(listed) + commit := strings.TrimSuffix(listed, ext) + requested, ok := strings.CutSuffix(filename, ext) + if !branch || !ok || !commitPattern.MatchString(commit) || !commitPattern.MatchString(requested) { + return "", false + } + idx := strings.LastIndex(listedURL, commit) + return listedURL[:idx] + requested + listedURL[idx+len(commit):], true +} + +// ShaQueryCommit returns the commit an archive name passes in a sha query +// parameter. GitLab names the archive of every commit archive.zip and picks +// the commit that way. +func ShaQueryCommit(filename string) (string, bool) { + _, query, ok := strings.Cut(filename, "?") + if !ok { + return "", false + } + values, err := url.ParseQuery(query) + if err != nil { + return "", false + } + commit := values.Get("sha") + return commit, commitPattern.MatchString(commit) +} + +// FindDist returns the upstream dist of version from Composer metadata, with an empty URL when the package has no such version or it has +// no dist URL. It walks the version list in place, expanding only as much of +// the minified format as it needs, since it runs on each archive the proxy +// has not cached yet. +func FindDist(body []byte, packageName, version string) (Dist, error) { + format, _, err := jsonscan.LookupString(body, "minified") + if err != nil { + return Dist{}, err + } + versions, err := jsonscan.Lookup(body, "packages", packageName) + if err != nil || len(versions) == 0 || versions[0] != '[' { + return Dist{}, err + } + minified := format == Minified + + fields := newFields() + var dist Dist + err = jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, DevReset) { + fields.reset() + return nil + } + if entry[0] != '{' { + return nil + } + if !minified { + fields.reset() + } + if err := fields.apply(entry, minified); err != nil { + return err + } + if !jsonscan.StringIs(fields.get("version"), version) { + return nil + } + distRaw := fields.get("dist") + dist.URL, _, _ = jsonscan.LookupString(distRaw, "url") + if dist.URL != "" { + dist.Type, _, _ = jsonscan.LookupString(distRaw, "type") + dist.Shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") + dist.Published = fields.get("time") + return jsonscan.ErrStop + } + return nil + }) + if errors.Is(err, jsonscan.ErrStop) { + err = nil + } + return dist, err +} + +// PushedVersion reports whether version is one of the pushed ones. Tags are +// listed with and without a leading v, and both name one version. +func PushedVersion(local []LocalVersion, version string) bool { + for _, e := range local { + if strings.TrimPrefix(e.Version, "v") == strings.TrimPrefix(version, "v") { + return true + } + } + return false +} + +// LocalDocument is a metadata document listing only the pushed versions. +func LocalDocument(packageName string, local []LocalVersion) ([]byte, error) { + entries := make([]json.RawMessage, 0, len(local)) + for _, e := range local { + entries = append(entries, e.Entry) + } + return json.Marshal(map[string]any{"packages": map[string]any{packageName: entries}}) +} + +// fields holds one version's fields as Composer sees them after +// expanding the minified format: raw values under their decoded names, in the +// order the names first appeared. A removed field keeps its slot with a nil +// value, so the order stays stable when it comes back. +type fields struct { + fields []field + index map[string]int +} + +type field struct { + name string + key, value []byte +} + +func newFields() *fields { + return &fields{index: map[string]int{}} +} + +func (f *fields) reset() { + f.fields = f.fields[:0] + clear(f.index) +} + +// get returns the raw value of the named field, or nil when it is not set. +func (f *fields) get(name string) []byte { + if i, ok := f.index[name]; ok { + return f.fields[i].value + } + return nil +} + +// string returns the named field when it is a string, or "". +func (f *fields) string(name string) string { + raw := f.get(name) + if len(raw) == 0 || raw[0] != '"' { + return "" + } + var s string + _ = json.Unmarshal(raw, &s) + return s +} + +// set stores value under name, written with the quoted key. A nil value +// removes the field. +func (f *fields) set(name string, key, value []byte) { + if i, ok := f.index[name]; ok { + f.fields[i].key, f.fields[i].value = key, value + return + } + f.index[name] = len(f.fields) + f.fields = append(f.fields, field{name: name, key: key, value: value}) +} + +// apply overlays the members of a version entry. In minified metadata the +// value "__unset" removes the field instead. +func (f *fields) apply(entry []byte, minified bool) error { + return jsonscan.EachMember(entry, func(m jsonscan.Member) error { + key, value := m.Key(entry), m.Value(entry) + if minified && jsonscan.StringIs(value, unset) { + value = nil + } + if bytes.IndexByte(key, '\\') < 0 { + if i, ok := f.index[string(key[1:len(key)-1])]; ok { + f.fields[i].key, f.fields[i].value = key, value + return nil + } + } + name, err := jsonscan.Key(key) + if err != nil { + return err + } + f.set(name, key, value) + return nil + }) +} diff --git a/internal/metadata/composer/metadata_test.go b/internal/metadata/composer/metadata_test.go new file mode 100644 index 00000000..5bc561b5 --- /dev/null +++ b/internal/metadata/composer/metadata_test.go @@ -0,0 +1,144 @@ +package composer + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// dayCooldown holds back versions published within a day of the time a +// rewrite is evaluated at, so tests can tell that time from the clock. +type dayCooldown struct{ enabled bool } + +func (c dayCooldown) Enabled() bool { return c.enabled } + +func (dayCooldown) Evaluate(_, _ string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + return cooldown.Decision{Allowed: evaluatedAt.Sub(publishedAt) >= 24*time.Hour} +} + +// A minified document: the second entry inherits the first one's fields. +const minifiedDocument = `{"minified": "composer/2.0", "packages": {"acme/tool": [ + {"name": "acme/tool", "version": "2.0.0", "time": "2026-01-10T12:00:00+00:00", + "dist": {"type": "zip", "url": "https://codeload.example/acme/tool/zip/bbb", "shasum": ""}}, + {"version": "1.0.0", "time": "2026-01-01T00:00:00+00:00", + "dist": {"type": "zip", "url": "https://codeload.example/acme/tool/zip/aaa", "shasum": ""}} +]}}` + +func testOptions() Options { + return Options{ + ProxyURL: "https://proxy.example", + Cooldown: dayCooldown{enabled: true}, + EvaluatedAt: time.Date(2026, 1, 11, 0, 0, 0, 0, time.UTC), + } +} + +// Cooldown is decided at EvaluatedAt and reported; the version that stays +// still carries the fields it inherited from the one left out. +func TestRewrite(t *testing.T) { + o := testOptions() + var filtered []string + o.OnCooldownFilter = func(name, version string, at time.Time) { + filtered = append(filtered, name+"@"+version+" "+at.UTC().Format(time.RFC3339)) + } + out, kept, err := Rewrite([]byte(minifiedDocument), o) + if err != nil { + t.Fatalf("Rewrite: %v", err) + } + if !slices.Equal(kept, []string{"acme/tool\x001.0.0"}) { + t.Errorf("kept = %q", kept) + } + if !slices.Equal(filtered, []string{"acme/tool@2.0.0 2026-01-10T12:00:00Z"}) { + t.Errorf("filtered = %v", filtered) + } + var doc struct { + Packages map[string][]map[string]any `json:"packages"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + versions := doc.Packages["acme/tool"] + if len(versions) != 1 || versions[0]["name"] != "acme/tool" || versions[0]["version"] != "1.0.0" { + t.Fatalf("versions = %v", versions) + } + dist, _ := versions[0]["dist"].(map[string]any) + if dist["url"] != "https://proxy.example/composer/files/acme/tool/1.0.0/aaa.zip" { + t.Errorf("dist url = %v", dist["url"]) + } +} + +// A disabled or missing policy keeps every version. +func TestRewriteWithoutCooldown(t *testing.T) { + for _, policy := range []CooldownPolicy{nil, dayCooldown{enabled: false}} { + o := testOptions() + o.Cooldown = policy + if _, kept, err := Rewrite([]byte(minifiedDocument), o); err != nil || len(kept) != 2 { + t.Errorf("policy %v: kept = %q, %v; want both versions", policy, kept, err) + } + } +} + +// Pushed versions take the place of upstream ones, and a routed package's +// versions don't report installs to the default repository. +func TestRewriteLocalAndRouted(t *testing.T) { + o := testOptions() + o.Cooldown = nil + o.LocalPackage = "acme/tool" + o.LocalVersions = []LocalVersion{{Version: "v1.0.0", Entry: json.RawMessage(`{"name": "acme/tool", "version": "1.0.0", "dist": {"type": "zip", "url": "https://proxy.example/composer/files/acme/tool/1.0.0/local.zip"}}`)}} + o.Routed = func(name string) bool { return name == "acme/tool" } + out, kept, err := Rewrite([]byte(minifiedDocument), o) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(kept, []string{"acme/tool\x002.0.0", "acme/tool\x00v1.0.0"}) { + t.Errorf("kept = %q", kept) + } + if !strings.Contains(string(out), "local.zip") || strings.Contains(string(out), "zip/aaa") { + t.Errorf("the pushed 1.0.0 did not replace the upstream one: %s", out) + } + if !strings.Contains(string(out), `"notification-url":""`) { + t.Errorf("a routed package's versions keep the default notification URL: %s", out) + } + + doc, err := LocalDocument("acme/other", o.LocalVersions) + if err != nil || !strings.Contains(string(doc), `"acme/other"`) { + t.Errorf("LocalDocument = %s, %v", doc, err) + } +} + +func TestFindDist(t *testing.T) { + dist, err := FindDist([]byte(minifiedDocument), "acme/tool", "1.0.0") + if err != nil || dist.URL != "https://codeload.example/acme/tool/zip/aaa" || dist.Type != "zip" || + string(dist.Published) != `"2026-01-01T00:00:00+00:00"` { + t.Errorf("FindDist = %+v, %v", dist, err) + } + if dist, err := FindDist([]byte(minifiedDocument), "acme/tool", "9.9.9"); err != nil || dist.URL != "" { + t.Errorf("missing version = %+v, %v", dist, err) + } +} + +func TestDistURLs(t *testing.T) { + if got, ok := ProxyDistURL("https://proxy.example", "acme/tool", "1.0.0", "https://codeload.example/acme/tool/zip/aaa", "zip"); !ok || + got != "https://proxy.example/composer/files/acme/tool/1.0.0/aaa.zip" { + t.Errorf("ProxyDistURL = %q, %v", got, ok) + } + if _, ok := ProxyDistURL("https://proxy.example", "tool", "1.0.0", "https://x/y.zip", "zip"); ok { + t.Error("ProxyDistURL accepted a name without a vendor") + } + commit := strings.Repeat("a", 40) + if !IsCommit(commit) || IsCommit("main") { + t.Error("IsCommit") + } + if c, ok := ShaQueryCommit("archive.zip?sha=" + commit); !ok || c != commit { + t.Errorf("ShaQueryCommit = %q, %v", c, ok) + } + if at, ok := PublishTime([]byte(`"2026-01-01T00:00:00+00:00"`)); !ok || !at.Equal(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) { + t.Errorf("PublishTime = %v, %v", at, ok) + } + if !PushedVersion([]LocalVersion{{Version: "v2.0.0"}}, "2.0.0") || PushedVersion(nil, "2.0.0") { + t.Error("PushedVersion") + } +}