|
5 | 5 | "bytes" |
6 | 6 | "compress/gzip" |
7 | 7 | "encoding/json" |
| 8 | + "errors" |
| 9 | + "fmt" |
8 | 10 | "io" |
9 | 11 | "net/http" |
10 | 12 | "slices" |
@@ -374,53 +376,94 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { |
374 | 376 | // publishTime reads a package file's timestamp from the subdir's |
375 | 377 | // repodata.json, fetched with the cache key and encoding proxyCached uses. |
376 | 378 | // The repodata is indexed once per document (see publishIndex), since a |
377 | | -// subdir's index runs to tens of megabytes and an install downloads many |
378 | | -// files from it. |
| 379 | +// subdir's index runs to hundreds of megabytes and an install downloads many |
| 380 | +// files from it. A compressed copy over metadata_max_size can't be cached, |
| 381 | +// so it is streamed from upstream for each check, as handleRepodata does. |
379 | 382 | func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { |
380 | 383 | repodataPath := "/" + channel + "/" + arch + "/repodata.json" |
381 | 384 | cacheKey := h.metadataCacheKey(repodataPath) |
382 | 385 | res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) |
383 | | - if res.err != nil { |
384 | | - return time.Time{}, false, res.err |
| 386 | + var index map[string]time.Time |
| 387 | + var err error |
| 388 | + switch { |
| 389 | + case errors.Is(res.err, ErrMetadataTooLarge): |
| 390 | + index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) |
| 391 | + case res.err != nil: |
| 392 | + err = res.err |
| 393 | + default: |
| 394 | + index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) |
385 | 395 | } |
386 | | - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) |
387 | 396 | if err != nil { |
388 | 397 | return time.Time{}, false, err |
389 | 398 | } |
390 | 399 | published, ok := index[filename] |
391 | 400 | return published, ok, nil |
392 | 401 | } |
393 | 402 |
|
394 | | -// indexRepodata maps each package file in a repodata.json to its timestamp, |
395 | | -// leaving out files without one. |
| 403 | +// indexRepodata indexes a cached repodata.json body. |
396 | 404 | func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { |
| 405 | + var reader io.Reader = bytes.NewReader(body) |
397 | 406 | if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { |
398 | | - decoded, err := gzip.NewReader(bytes.NewReader(body)) |
| 407 | + decoded, err := gzip.NewReader(reader) |
399 | 408 | if err != nil { |
400 | 409 | return nil, err |
401 | 410 | } |
402 | | - body, err = h.proxy.ReadMetadata(decoded) |
403 | | - if err != nil { |
404 | | - return nil, err |
405 | | - } |
406 | | - } |
407 | | - type entry struct { |
408 | | - Timestamp float64 `json:"timestamp"` |
| 411 | + reader = decoded |
409 | 412 | } |
410 | | - var repodata struct { |
411 | | - Packages map[string]entry `json:"packages"` |
412 | | - PackagesConda map[string]entry `json:"packages.conda"` |
| 413 | + return readRepodataIndex(reader) |
| 414 | +} |
| 415 | + |
| 416 | +// streamRepodataIndex indexes repodata.json as it streams from upstream. |
| 417 | +func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { |
| 418 | + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) |
| 419 | + if err != nil { |
| 420 | + return nil, err |
413 | 421 | } |
414 | | - if err := json.Unmarshal(body, &repodata); err != nil { |
| 422 | + req.Header.Set(headerAcceptEncoding, "gzip") |
| 423 | + resp, err := h.proxy.HTTPClient.Do(req) |
| 424 | + if err != nil { |
415 | 425 | return nil, err |
416 | 426 | } |
417 | | - index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda)) |
418 | | - for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} { |
419 | | - for filename, e := range files { |
420 | | - if e.Timestamp != 0 { |
421 | | - index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) |
422 | | - } |
| 427 | + defer func() { _ = resp.Body.Close() }() |
| 428 | + if resp.StatusCode != http.StatusOK { |
| 429 | + return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) |
| 430 | + } |
| 431 | + var reader io.Reader = resp.Body |
| 432 | + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { |
| 433 | + decoded, err := gzip.NewReader(resp.Body) |
| 434 | + if err != nil { |
| 435 | + return nil, err |
| 436 | + } |
| 437 | + defer func() { _ = decoded.Close() }() |
| 438 | + reader = decoded |
| 439 | + } |
| 440 | + return readRepodataIndex(reader) |
| 441 | +} |
| 442 | + |
| 443 | +// readRepodataIndex maps each package file in a repodata.json to its |
| 444 | +// timestamp, leaving out files without one. It reads the document as it |
| 445 | +// streams, under the same bound as filterRepodata, so the decompressed size |
| 446 | +// is not limited by metadata_max_size. |
| 447 | +func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { |
| 448 | + index := make(map[string]time.Time) |
| 449 | + err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { |
| 450 | + if m.Key != "packages" && m.Key != "packages.conda" { |
| 451 | + return m.Copy() |
423 | 452 | } |
| 453 | + return m.Object(func(entry *jsonstream.Member) error { |
| 454 | + raw, err := entry.Raw() |
| 455 | + if err != nil { |
| 456 | + return err |
| 457 | + } |
| 458 | + var e repodataEntry |
| 459 | + if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { |
| 460 | + index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) |
| 461 | + } |
| 462 | + return nil |
| 463 | + }) |
| 464 | + }) |
| 465 | + if err != nil { |
| 466 | + return nil, err |
424 | 467 | } |
425 | 468 | return index, nil |
426 | 469 | } |
|
0 commit comments