Skip to content

Commit 1a9bda2

Browse files
committed
Conda reads repodata as a stream for the download check, under the same bound as the repodata filter, so a repodata.json over metadata_max_size no longer lets a package in cooldown through. If even the compressed copy is too large to cache, the check streams it from upstream.
1 parent 9577472 commit 1a9bda2

2 files changed

Lines changed: 116 additions & 25 deletions

File tree

‎internal/handler/conda.go‎

Lines changed: 68 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"bytes"
66
"compress/gzip"
77
"encoding/json"
8+
"errors"
9+
"fmt"
810
"io"
911
"net/http"
1012
"slices"
@@ -374,53 +376,94 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string {
374376
// publishTime reads a package file's timestamp from the subdir's
375377
// repodata.json, fetched with the cache key and encoding proxyCached uses.
376378
// The repodata is indexed once per document (see publishIndex), since a
377-
// subdir's index runs to tens of megabytes and an install downloads many
378-
// files from it.
379+
// subdir's index runs to hundreds of megabytes and an install downloads many
380+
// files from it. A compressed copy over metadata_max_size can't be cached,
381+
// so it is streamed from upstream for each check, as handleRepodata does.
379382
func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) {
380383
repodataPath := "/" + channel + "/" + arch + "/repodata.json"
381384
cacheKey := h.metadataCacheKey(repodataPath)
382385
res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil)
383-
if res.err != nil {
384-
return time.Time{}, false, res.err
386+
var index map[string]time.Time
387+
var err error
388+
switch {
389+
case errors.Is(res.err, ErrMetadataTooLarge):
390+
index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath)
391+
case res.err != nil:
392+
err = res.err
393+
default:
394+
index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata)
385395
}
386-
index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata)
387396
if err != nil {
388397
return time.Time{}, false, err
389398
}
390399
published, ok := index[filename]
391400
return published, ok, nil
392401
}
393402

394-
// indexRepodata maps each package file in a repodata.json to its timestamp,
395-
// leaving out files without one.
403+
// indexRepodata indexes a cached repodata.json body.
396404
func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) {
405+
var reader io.Reader = bytes.NewReader(body)
397406
if strings.EqualFold(strings.TrimSpace(encoding), "gzip") {
398-
decoded, err := gzip.NewReader(bytes.NewReader(body))
407+
decoded, err := gzip.NewReader(reader)
399408
if err != nil {
400409
return nil, err
401410
}
402-
body, err = h.proxy.ReadMetadata(decoded)
403-
if err != nil {
404-
return nil, err
405-
}
406-
}
407-
type entry struct {
408-
Timestamp float64 `json:"timestamp"`
411+
reader = decoded
409412
}
410-
var repodata struct {
411-
Packages map[string]entry `json:"packages"`
412-
PackagesConda map[string]entry `json:"packages.conda"`
413+
return readRepodataIndex(reader)
414+
}
415+
416+
// streamRepodataIndex indexes repodata.json as it streams from upstream.
417+
func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) {
418+
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil)
419+
if err != nil {
420+
return nil, err
413421
}
414-
if err := json.Unmarshal(body, &repodata); err != nil {
422+
req.Header.Set(headerAcceptEncoding, "gzip")
423+
resp, err := h.proxy.HTTPClient.Do(req)
424+
if err != nil {
415425
return nil, err
416426
}
417-
index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda))
418-
for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} {
419-
for filename, e := range files {
420-
if e.Timestamp != 0 {
421-
index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0)
422-
}
427+
defer func() { _ = resp.Body.Close() }()
428+
if resp.StatusCode != http.StatusOK {
429+
return nil, fmt.Errorf("repodata returned %d", resp.StatusCode)
430+
}
431+
var reader io.Reader = resp.Body
432+
if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") {
433+
decoded, err := gzip.NewReader(resp.Body)
434+
if err != nil {
435+
return nil, err
436+
}
437+
defer func() { _ = decoded.Close() }()
438+
reader = decoded
439+
}
440+
return readRepodataIndex(reader)
441+
}
442+
443+
// readRepodataIndex maps each package file in a repodata.json to its
444+
// timestamp, leaving out files without one. It reads the document as it
445+
// streams, under the same bound as filterRepodata, so the decompressed size
446+
// is not limited by metadata_max_size.
447+
func readRepodataIndex(r io.Reader) (map[string]time.Time, error) {
448+
index := make(map[string]time.Time)
449+
err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error {
450+
if m.Key != "packages" && m.Key != "packages.conda" {
451+
return m.Copy()
423452
}
453+
return m.Object(func(entry *jsonstream.Member) error {
454+
raw, err := entry.Raw()
455+
if err != nil {
456+
return err
457+
}
458+
var e repodataEntry
459+
if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 {
460+
index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0)
461+
}
462+
return nil
463+
})
464+
})
465+
if err != nil {
466+
return nil, err
424467
}
425468
return index, nil
426469
}

‎internal/handler/download_cooldown_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,3 +417,51 @@ func TestPublishIndexBuildsOncePerDocument(t *testing.T) {
417417
}
418418
}
419419
}
420+
421+
// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is
422+
// withheld when its repodata is over metadata_max_size, as the repodata
423+
// endpoint hides it: once only decompressed, and once already compressed.
424+
func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) {
425+
plain := largeRepodata(500)
426+
compressed := gzipPayload(t, plain)
427+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
428+
w.Header().Set(headerContentEncoding, "gzip")
429+
_, _ = w.Write(compressed)
430+
}))
431+
t.Cleanup(upstream.Close)
432+
433+
for _, tt := range []struct {
434+
name string
435+
limit int64
436+
}{
437+
{"decompressed over the limit", int64(len(compressed)) + 1},
438+
{"compressed over the limit", int64(len(compressed)) - 1},
439+
} {
440+
t.Run(tt.name, func(t *testing.T) {
441+
proxy, db, store, fetcher := setupTestProxy(t)
442+
proxy.HTTPClient = upstream.Client()
443+
proxy.CacheMetadata = true
444+
proxy.MetadataTTL = time.Hour
445+
proxy.MetadataMaxSize = tt.limit
446+
fetcher.fetchErr = errors.New("artifact must be served from the cache")
447+
for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} {
448+
seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file)
449+
}
450+
proxy.Cooldown = &cooldown.Config{Default: "3d"}
451+
h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes()
452+
453+
w := httptest.NewRecorder()
454+
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil))
455+
if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") {
456+
t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code)
457+
}
458+
for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} {
459+
w := httptest.NewRecorder()
460+
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil))
461+
if w.Code != want {
462+
t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String())
463+
}
464+
}
465+
})
466+
}
467+
}

0 commit comments

Comments
 (0)