Skip to content

Commit 7a76983

Browse files
committed
Conda now checks each package file against its own timestamp, because builds of one version differ. Conda repodata and helm indexes are parsed once per cached document instead of on every download.
1 parent 2f89ce5 commit 7a76983

5 files changed

Lines changed: 234 additions & 59 deletions

File tree

‎internal/handler/conda.go‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
113113
h.proxy.Logger.Info("conda download request",
114114
"channel", channel, "arch", arch, "name", name, "version", version)
115115

116+
// Builds and subdirs of one version each have their own timestamp, so
117+
// the check is per file and keeps no time in the shared versions row.
116118
if h.proxy.withholdInCooldown(w, downloadVersion{
117-
ecosystem: "conda", policyName: name, storedName: packageName, version: version,
119+
ecosystem: "conda", policyName: name, version: version,
118120
lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) },
119121
}) {
120122
return
@@ -306,40 +308,57 @@ func (h *CondaHandler) inCooldown(filename string, raw []byte) bool {
306308
}
307309

308310
// publishTime reads a package file's timestamp from the subdir's
309-
// repodata.json. It shares proxyCached's cache entry and encoding, so the
310-
// repodata a client just fetched is not downloaded again, and reads the one
311-
// entry in place rather than decoding the whole index.
311+
// repodata.json, fetched with the cache key and encoding proxyCached uses.
312+
// The repodata is indexed once per document (see publishIndex), since a
313+
// subdir's index runs to tens of megabytes and an install downloads many
314+
// files from it.
312315
func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) {
313316
repodataPath := "/" + channel + "/" + arch + "/repodata.json"
314317
cacheKey := strings.ReplaceAll(strings.TrimPrefix(repodataPath, "/"), "/", "_")
315-
body, _, encoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "gzip", nil, "*/*")
318+
res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil)
319+
if res.err != nil {
320+
return time.Time{}, false, res.err
321+
}
322+
index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata)
316323
if err != nil {
317324
return time.Time{}, false, err
318325
}
326+
published, ok := index[filename]
327+
return published, ok, nil
328+
}
329+
330+
// indexRepodata maps each package file in a repodata.json to its timestamp,
331+
// leaving out files without one.
332+
func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) {
319333
if strings.EqualFold(strings.TrimSpace(encoding), "gzip") {
320334
decoded, err := gzip.NewReader(bytes.NewReader(body))
321335
if err != nil {
322-
return time.Time{}, false, err
336+
return nil, err
323337
}
324338
body, err = h.proxy.ReadMetadata(decoded)
325339
if err != nil {
326-
return time.Time{}, false, err
340+
return nil, err
327341
}
328342
}
329-
330-
key := "packages"
331-
if strings.HasSuffix(filename, ".conda") {
332-
key = "packages.conda"
343+
type entry struct {
344+
Timestamp float64 `json:"timestamp"`
333345
}
334-
raw, err := lookupJSON(body, key, filename, "timestamp")
335-
if err != nil || len(raw) == 0 {
336-
return time.Time{}, false, err
346+
var repodata struct {
347+
Packages map[string]entry `json:"packages"`
348+
PackagesConda map[string]entry `json:"packages.conda"`
349+
}
350+
if err := json.Unmarshal(body, &repodata); err != nil {
351+
return nil, err
337352
}
338-
var ts float64
339-
if err := json.Unmarshal(raw, &ts); err != nil || ts == 0 {
340-
return time.Time{}, false, nil
353+
index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda))
354+
for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} {
355+
for filename, e := range files {
356+
if e.Timestamp != 0 {
357+
index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0)
358+
}
359+
}
341360
}
342-
return time.Unix(int64(ts)/condaTimestampDivisor, 0), true, nil
361+
return index, nil
343362
}
344363

345364
// proxyCached forwards a metadata request with caching.

‎internal/handler/download_cooldown.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ package handler
22

33
import (
44
"net/http"
5+
"sync"
56
"time"
7+
8+
"golang.org/x/sync/singleflight"
69
)
710

811
// publishTimeLookup reads a version's publish time from upstream metadata. ok
@@ -80,3 +83,62 @@ func (p *Proxy) versionInCooldown(d downloadVersion) bool {
8083

8184
return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published)
8285
}
86+
87+
// maxPublishIndexes bounds how many documents publishIndexCache keeps an
88+
// index for.
89+
const maxPublishIndexes = 64
90+
91+
// publishIndexCache keeps, for each large metadata document, the publish
92+
// times a download check reads from it, so a document is decompressed and
93+
// parsed once per version from upstream rather than on every download.
94+
// Concurrent checks for a document not indexed yet share one parse.
95+
type publishIndexCache struct {
96+
mu sync.Mutex
97+
entries map[string]publishIndexEntry
98+
group singleflight.Group
99+
}
100+
101+
type publishIndexEntry struct {
102+
digest string
103+
index any
104+
}
105+
106+
// publishIndex returns build's index of res, reusing the one built for the
107+
// same document under key. res.digest identifies the document; without one,
108+
// as when the metadata cache is off, the index is built for this call alone.
109+
func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) {
110+
c := &p.publishIndexes
111+
if res.digest == "" {
112+
return build(res.body, res.contentEncoding)
113+
}
114+
c.mu.Lock()
115+
if e, ok := c.entries[key]; ok && e.digest == res.digest {
116+
c.mu.Unlock()
117+
return e.index.(map[string]V), nil
118+
}
119+
c.mu.Unlock()
120+
121+
v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) {
122+
index, err := build(res.body, res.contentEncoding)
123+
if err != nil {
124+
return nil, err
125+
}
126+
c.mu.Lock()
127+
defer c.mu.Unlock()
128+
if c.entries == nil {
129+
c.entries = make(map[string]publishIndexEntry)
130+
}
131+
if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes {
132+
for evict := range c.entries {
133+
delete(c.entries, evict)
134+
break
135+
}
136+
}
137+
c.entries[key] = publishIndexEntry{digest: res.digest, index: index}
138+
return index, nil
139+
})
140+
if err != nil {
141+
return nil, err
142+
}
143+
return v.(map[string]V), nil
144+
}

‎internal/handler/download_cooldown_test.go‎

Lines changed: 90 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ const (
2626
// is always named demo, so the policy key is pkg:{ecosystem}/demo.
2727
type downloadCooldownCase struct {
2828
ecosystem string
29-
// storedName is the name the artifact is cached under; empty when the
30-
// ecosystem keeps no publish time in the versions table.
31-
storedName string
29+
// cachedName is the name the artifact is cached under, and keepsTime
30+
// whether the publish time is kept in its versions row.
31+
cachedName string
32+
keepsTime bool
3233
// artifactVersion maps a release to the version the artifact is cached
3334
// under, and filename and path to its file and download path.
3435
artifactVersion func(version string) string
@@ -48,7 +49,8 @@ func downloadCooldownCases() []downloadCooldownCase {
4849
return []downloadCooldownCase{
4950
{
5051
ecosystem: "cargo",
51-
storedName: "demo",
52+
cachedName: "demo",
53+
keepsTime: true,
5254
artifactVersion: sameVersion,
5355
filename: func(v string) string { return "demo-" + v + ".crate" },
5456
path: func(v string) string { return "/crates/demo/" + v + "/download" },
@@ -57,7 +59,8 @@ func downloadCooldownCases() []downloadCooldownCase {
5759
},
5860
{
5961
ecosystem: "gem",
60-
storedName: "demo",
62+
cachedName: "demo",
63+
keepsTime: true,
6164
// Platform gems carry the platform in the filename and the
6265
// cached version, but not in the versions API number.
6366
artifactVersion: func(v string) string { return v + "-x86_64-linux" },
@@ -68,7 +71,8 @@ func downloadCooldownCases() []downloadCooldownCase {
6871
},
6972
{
7073
ecosystem: "hex",
71-
storedName: "demo",
74+
cachedName: "demo",
75+
keepsTime: true,
7276
artifactVersion: sameVersion,
7377
filename: func(v string) string { return "demo-" + v + ".tar" },
7478
path: func(v string) string { return "/tarballs/demo-" + v + ".tar" },
@@ -79,7 +83,8 @@ func downloadCooldownCases() []downloadCooldownCase {
7983
},
8084
{
8185
ecosystem: "pub",
82-
storedName: "demo",
86+
cachedName: "demo",
87+
keepsTime: true,
8388
artifactVersion: sameVersion,
8489
filename: func(v string) string { return "demo-" + v + ".tar.gz" },
8590
path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" },
@@ -88,9 +93,10 @@ func downloadCooldownCases() []downloadCooldownCase {
8893
},
8994
{
9095
// Conda caches under {channel}/{name}, but the policy is keyed by
91-
// the name alone, as the repodata filter keys it.
96+
// the name alone, as the repodata filter keys it. Each build has
97+
// its own timestamp, so none is kept in the version's row.
9298
ecosystem: "conda",
93-
storedName: "conda-forge/demo",
99+
cachedName: "conda-forge/demo",
94100
artifactVersion: sameVersion,
95101
filename: func(v string) string { return "demo-" + v + "-py_0.conda" },
96102
path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" },
@@ -102,6 +108,7 @@ func downloadCooldownCases() []downloadCooldownCase {
102108
{
103109
// Helm caches charts under the repository and digest, so the
104110
// chart name and time come from the index on every check.
111+
cachedName: "stable",
105112
ecosystem: "helm",
106113
artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) },
107114
filename: func(v string) string { return "demo-" + v + ".tgz" },
@@ -217,12 +224,11 @@ func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[stri
217224
func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) {
218225
t.Helper()
219226
db, store := proxy.DB, proxy.Storage.(*mockStorage)
220-
name := c.storedName
221227
content := "artifact " + version
222228
if c.ecosystem == "helm" {
223-
name, content = "stable", chartContent(version)
229+
content = chartContent(version)
224230
}
225-
seedPackage(t, db, store, c.ecosystem, name, c.artifactVersion(version), c.filename(version), content)
231+
seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content)
226232
}
227233

228234
// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's
@@ -267,10 +273,10 @@ func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) {
267273
if w.Code == http.StatusOK && w.Body.String() == "" {
268274
t.Error("served an empty body")
269275
}
270-
if c.storedName == "" || times[version].IsZero() {
276+
if !c.keepsTime || times[version].IsZero() {
271277
return
272278
}
273-
ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(version)))
279+
ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version)))
274280
if err != nil || ver == nil || !ver.PublishedAt.Valid {
275281
t.Fatalf("publish time not stored: %+v, %v", ver, err)
276282
}
@@ -291,16 +297,16 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) {
291297
t.Cleanup(upstream.Close)
292298

293299
for _, c := range downloadCooldownCases() {
294-
if c.storedName == "" {
300+
if !c.keepsTime {
295301
continue
296302
}
297303
t.Run(c.ecosystem, func(t *testing.T) {
298304
proxy, db, _, fetcher := setupTestProxy(t)
299305
proxy.HTTPClient = upstream.Client()
300306
fetcher.fetchErr = errors.New("artifact must be served from the cache")
301307
c.seedCachedArtifact(t, proxy, cooldownRecentVersion)
302-
versionPURL := canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(cooldownRecentVersion))
303-
packagePURL := canonicalPackagePURL(c.ecosystem, c.storedName)
308+
versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion))
309+
packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName)
304310
if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil {
305311
t.Fatalf("seeding publish time: %v", err)
306312
}
@@ -320,7 +326,7 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) {
320326
// needs the index to learn the chart name, so it is left out.
321327
func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) {
322328
for _, c := range downloadCooldownCases() {
323-
if c.storedName == "" {
329+
if c.ecosystem == "helm" {
324330
continue
325331
}
326332
t.Run(c.ecosystem, func(t *testing.T) {
@@ -345,3 +351,69 @@ func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) {
345351
})
346352
}
347353
}
354+
355+
// TestCondaDownloadCooldownPerBuild checks that builds of one version are
356+
// judged by their own timestamps, whichever is requested first: a rebuild
357+
// inside the window is withheld and an old build stays available.
358+
func TestCondaDownloadCooldownPerBuild(t *testing.T) {
359+
now := time.Now()
360+
repodata := map[string]any{"packages.conda": map[string]any{
361+
"demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()},
362+
"demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()},
363+
}}
364+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
365+
_ = json.NewEncoder(w).Encode(repodata)
366+
}))
367+
t.Cleanup(upstream.Close)
368+
369+
builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound}
370+
for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} {
371+
t.Run(strings.Join(order, ","), func(t *testing.T) {
372+
proxy, db, store, fetcher := setupTestProxy(t)
373+
proxy.HTTPClient = upstream.Client()
374+
fetcher.fetchErr = errors.New("artifact must be served from the cache")
375+
for build := range builds {
376+
filename := "demo-1.0.0-" + build + ".conda"
377+
seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build)
378+
}
379+
proxy.Cooldown = &cooldown.Config{Default: "7d"}
380+
h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes()
381+
382+
for _, build := range order {
383+
w := httptest.NewRecorder()
384+
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil))
385+
if w.Code != builds[build] {
386+
t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build])
387+
}
388+
}
389+
})
390+
}
391+
}
392+
393+
// TestPublishIndexBuildsOncePerDocument checks that an index is reused for
394+
// the same document, rebuilt for a new one, and not kept without a digest.
395+
func TestPublishIndexBuildsOncePerDocument(t *testing.T) {
396+
proxy, _, _, _ := setupTestProxy(t)
397+
builds := 0
398+
build := func(body []byte, _ string) (map[string]bool, error) {
399+
builds++
400+
return map[string]bool{string(body): true}, nil
401+
}
402+
steps := []struct {
403+
res metadataResult
404+
want string
405+
wantBuilds int
406+
}{
407+
{metadataResult{body: []byte("a"), digest: "d1"}, "a", 1},
408+
{metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1},
409+
{metadataResult{body: []byte("b"), digest: "d2"}, "b", 2},
410+
{metadataResult{body: []byte("c")}, "c", 3},
411+
{metadataResult{body: []byte("c")}, "c", 4},
412+
}
413+
for i, step := range steps {
414+
got, err := publishIndex(proxy, "key", step.res, build)
415+
if err != nil || !got[step.want] || builds != step.wantBuilds {
416+
t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds)
417+
}
418+
}
419+
}

‎internal/handler/handler.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,10 @@ type Proxy struct {
222222
// Nil leaves every request to rewrite its own copy.
223223
rewrites *rewriteCache
224224

225+
// publishIndexes keeps publish times read out of large metadata
226+
// documents for the download cooldown check; see publishIndex.
227+
publishIndexes publishIndexCache
228+
225229
// now is the clock rewritten metadata is dated by, and started the time
226230
// the process started; see metadataDate.
227231
now func() time.Time

0 commit comments

Comments
 (0)