Skip to content

Commit 0f85f6d

Browse files
authored
Merge pull request #690 from simonchrz/fix/debian-conan-cache-keys
Key Debian and Conan archives by their whole path
2 parents b43e6a8 + 3e50f45 commit 0f85f6d

7 files changed

Lines changed: 274 additions & 28 deletions

File tree

‎internal/handler/conan.go‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package handler
22

33
import (
44
"net/http"
5+
"slices"
56
"strings"
67

78
"github.com/git-pkgs/proxy/internal/retention"
@@ -79,6 +80,11 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request)
7980
revision := r.PathValue("revision")
8081
filename := r.PathValue("filename")
8182

83+
if conanPartHasSlash(user, channel, revision, filename) {
84+
http.Error(w, "invalid path", http.StatusBadRequest)
85+
return
86+
}
87+
8288
// Only the archives are artifacts; the rest goes through the metadata cache.
8389
if !h.shouldCacheFile(filename) {
8490
h.proxyCached(w, r)
@@ -89,7 +95,13 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request)
8995
"name", name, "version", version, "user", user, "channel", channel, "filename", filename)
9096

9197
upstreamURL := h.upstreamURL + r.URL.Path
92-
storageFilename := conanStorageFilename(user, channel, revision+"_"+filename)
98+
storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version,
99+
conanStorageFilename(user, channel, revision, filename),
100+
conanLegacyStorageFilename(user, channel, revision+"_"+filename), upstreamURL)
101+
if err != nil {
102+
h.proxy.serveArtifactError(w, err, "failed to fetch file")
103+
return
104+
}
93105

94106
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL)
95107
if err != nil {
@@ -111,6 +123,11 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
111123
pkgrev := r.PathValue("pkgrev")
112124
filename := r.PathValue("filename")
113125

126+
if conanPartHasSlash(user, channel, revision, pkgref, pkgrev, filename) {
127+
http.Error(w, "invalid path", http.StatusBadRequest)
128+
return
129+
}
130+
114131
// Only the archives are artifacts; the rest goes through the metadata cache.
115132
if !h.shouldCacheFile(filename) {
116133
h.proxyCached(w, r)
@@ -121,7 +138,13 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
121138
"name", name, "version", version, "pkgref", pkgref, "filename", filename)
122139

123140
upstreamURL := h.upstreamURL + r.URL.Path
124-
storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename)
141+
storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version,
142+
conanStorageFilename(user, channel, revision, "packages", pkgref, pkgrev, filename),
143+
conanLegacyStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename), upstreamURL)
144+
if err != nil {
145+
h.proxy.serveArtifactError(w, err, "failed to fetch file")
146+
return
147+
}
125148

126149
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL)
127150
if err != nil {
@@ -133,9 +156,25 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request)
133156
}
134157

135158
// conanStorageFilename names a cached archive. Archives are stored under
136-
// pkg:conan/{name}@{version}, so the file name carries the revisions and, when
137-
// set, user and channel, to keep every recipe and binary apart.
138-
func conanStorageFilename(user, channel, file string) string {
159+
// pkg:conan/{name}@{version}, so the file name carries user, channel and the
160+
// revisions, to keep every recipe and binary apart. The handlers refuse a
161+
// part that contains a slash, so joining them with one keeps different
162+
// references apart.
163+
func conanStorageFilename(user, channel string, parts ...string) string {
164+
return strings.Join(append([]string{user, channel}, parts...), "/")
165+
}
166+
167+
// conanPartHasSlash reports whether a path value contains a slash, which it
168+
// can only get from an encoded one (%2F) in the request.
169+
func conanPartHasSlash(parts ...string) bool {
170+
return slices.ContainsFunc(parts, func(part string) bool { return strings.Contains(part, "/") })
171+
}
172+
173+
// conanLegacyStorageFilename is the name archives were cached under before
174+
// their parts were joined with slashes. Underscores, which revisions, user
175+
// and channel can contain, joined them, so it is only read for the URL an
176+
// entry was fetched from.
177+
func conanLegacyStorageFilename(user, channel, file string) string {
139178
if user == "_" && channel == "_" {
140179
return file
141180
}

‎internal/handler/conan_test.go‎

Lines changed: 108 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package handler
22

33
import (
4+
"context"
45
"io"
56
"log/slog"
67
"net/http"
@@ -9,6 +10,8 @@ import (
910
"sync/atomic"
1011
"testing"
1112
"time"
13+
14+
"github.com/git-pkgs/registries/fetch"
1215
)
1316

1417
const testProxyURL = "http://localhost:8080"
@@ -302,13 +305,13 @@ func TestConanV2FileRoutesCacheArchives(t *testing.T) {
302305
filename string
303306
}{
304307
{"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz",
305-
"rrev1_conan_sources.tgz"},
308+
"_/_/rrev1/conan_sources.tgz"},
306309
{"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz",
307-
"rrev1_pkgid1_prev1_conan_package.tgz"},
310+
"_/_/rrev1/packages/pkgid1/prev1/conan_package.tgz"},
308311
{"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz",
309-
"rrev1_conan_export.txz"},
312+
"_/_/rrev1/conan_export.txz"},
310313
{"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst",
311-
"rrev1_pkgid1_prev1_conan_package.tzst"},
314+
"_/_/rrev1/packages/pkgid1/prev1/conan_package.tzst"},
312315
}
313316

314317
for _, tt := range tests {
@@ -362,9 +365,9 @@ func TestConanUserChannelKeptApart(t *testing.T) {
362365

363366
for _, ref := range []struct{ path, body, filename string }{
364367
{"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz",
365-
"center sources", "rrev1_conan_sources.tgz"},
368+
"center sources", "_/_/rrev1/conan_sources.tgz"},
366369
{"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz",
367-
"acme sources", "acme@stable_rrev1_conan_sources.tgz"},
370+
"acme sources", "acme/stable/rrev1/conan_sources.tgz"},
368371
} {
369372
fetcher.artifact = artifactBody(ref.body)
370373
fetcher.fetchCalled = false
@@ -651,3 +654,102 @@ func TestConanMetadataPreservesStatusCodes(t *testing.T) {
651654
upstream.Close()
652655
}
653656
}
657+
658+
const (
659+
conanRecipeLookalike = "/v2/conans/zlib/1.3/_/_/revisions/R_P_Q/files/conan_package.tgz"
660+
conanPackageFile = "/v2/conans/zlib/1.3/_/_/revisions/R/packages/P/revisions/Q/files/conan_package.tgz"
661+
)
662+
663+
// conanPathEchoServer answers every file with its own path, so each URL has
664+
// different bytes.
665+
func conanPathEchoServer(t *testing.T, available *atomic.Bool) *httptest.Server {
666+
t.Helper()
667+
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
668+
if !available.Load() {
669+
http.Error(w, "unavailable", http.StatusServiceUnavailable)
670+
return
671+
}
672+
_, _ = w.Write([]byte(r.URL.Path))
673+
}))
674+
t.Cleanup(upstream.Close)
675+
return upstream
676+
}
677+
678+
func serveConanRequest(h *ConanHandler, target string) *httptest.ResponseRecorder {
679+
w := httptest.NewRecorder()
680+
h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
681+
return w
682+
}
683+
684+
// A recipe revision with underscores must not share a cache entry with a
685+
// package whose revision, reference and package revision spell it.
686+
func TestConanFileKeysKeepPathFieldsApart(t *testing.T) {
687+
var available atomic.Bool
688+
available.Store(true)
689+
upstream := conanPathEchoServer(t, &available)
690+
proxy, _, _, _ := setupTestProxy(t)
691+
proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
692+
h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL)
693+
694+
for _, target := range []string{conanRecipeLookalike, conanPackageFile} {
695+
if w := serveConanRequest(h, target); w.Code != http.StatusOK || w.Body.String() != target {
696+
t.Errorf("GET %s = %d %q, want its own file", target, w.Code, w.Body.String())
697+
}
698+
}
699+
}
700+
701+
// A file cached under the name used before path fields were kept apart is
702+
// still served for the URL it was fetched from, and only for that URL.
703+
func TestConanLegacyFileEntryServedForItsURL(t *testing.T) {
704+
var available atomic.Bool
705+
available.Store(true)
706+
upstream := conanPathEchoServer(t, &available)
707+
proxy, _, _, _ := setupTestProxy(t)
708+
proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
709+
h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL)
710+
711+
legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "conan", "zlib", "1.3",
712+
"R_P_Q_conan_package.tgz", upstream.URL+conanRecipeLookalike)
713+
if err != nil {
714+
t.Fatalf("seed legacy entry: %v", err)
715+
}
716+
_ = legacy.Reader.Close()
717+
718+
if w := serveConanRequest(h, conanPackageFile); w.Body.String() != conanPackageFile {
719+
t.Errorf("package file = %d %q, want its own file, not the legacy entry", w.Code, w.Body.String())
720+
}
721+
available.Store(false)
722+
if w := serveConanRequest(h, conanRecipeLookalike); w.Code != http.StatusOK || w.Body.String() != conanRecipeLookalike {
723+
t.Errorf("recipe file with the upstream down = %d %q, want the legacy entry", w.Code, w.Body.String())
724+
}
725+
}
726+
727+
// A path value is decoded, so one holding an encoded slash could spell the
728+
// cache key of another reference. Those requests are refused.
729+
func TestConanFileRoutesRefuseEncodedSlash(t *testing.T) {
730+
var available atomic.Bool
731+
available.Store(true)
732+
upstream := conanPathEchoServer(t, &available)
733+
proxy, _, _, _ := setupTestProxy(t)
734+
proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
735+
h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL)
736+
737+
if w := serveConanRequest(h, conanPackageFile); w.Code != http.StatusOK || w.Body.String() != conanPackageFile {
738+
t.Fatalf("GET %s = %d %q, want its own file", conanPackageFile, w.Code, w.Body.String())
739+
}
740+
741+
const recipe, pkg = "/v2/conans/zlib/1.3/_/_/revisions/", "/v2/conans/zlib/1.3/_/_/revisions/R/packages/"
742+
for field, target := range map[string]string{
743+
"revision spelling a package file": recipe + "R%2Fpackages%2FP%2FQ/files/conan_package.tgz",
744+
"user": "/v2/conans/zlib/1.3/a%2Fb/_/revisions/R/files/conan_package.tgz",
745+
"channel": "/v2/conans/zlib/1.3/_/a%2Fb/revisions/R/files/conan_package.tgz",
746+
"recipe filename": recipe + "R/files/a%2Fconan_package.tgz",
747+
"package reference": pkg + "P%2FX/revisions/Q/files/conan_package.tgz",
748+
"package revision": pkg + "P/revisions/Q%2FX/files/conan_package.tgz",
749+
"package filename": pkg + "P/revisions/Q/files/a%2Fconan_package.tgz",
750+
} {
751+
if w := serveConanRequest(h, target); w.Code != http.StatusBadRequest {
752+
t.Errorf("%s: GET %s = %d %q, want 400", field, target, w.Code, w.Body.String())
753+
}
754+
}
755+
}

‎internal/handler/debian.go‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,12 @@ func (h *DebianHandler) handlePackageDownload(
169169
filename := path[strings.LastIndex(path, "/")+1:]
170170
downloadURL := fmt.Sprintf("%s/%s", upstreamURL, path)
171171

172-
cacheFilename := h.artifactCacheFilename(repository, filename)
172+
cacheFilename, err := h.proxy.cacheFilenameKeepingLegacy("deb", name, version,
173+
h.artifactCacheFilename(repository, path), h.legacyArtifactCacheFilename(repository, filename), downloadURL)
174+
if err != nil {
175+
h.proxy.serveArtifactError(w, err, "failed to fetch package")
176+
return
177+
}
173178

174179
h.proxy.Logger.Info("debian package download",
175180
"repository", repository,
@@ -233,10 +238,21 @@ func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL,
233238
return nil, ""
234239
}
235240

236-
// artifactCacheFilename scopes a package by repository, since the same
237-
// filename can hold different bytes in different archives. The main archive
238-
// keeps the unscoped filename its existing cache entries were stored under.
239-
func (h *DebianHandler) artifactCacheFilename(repository, filename string) string {
241+
// artifactCacheFilename names a package in the cache by its repository and
242+
// its path in it, since the same file name can hold different bytes in
243+
// different archives, and in different directories of one archive: Docker's
244+
// repository has a pool under each suite.
245+
func (h *DebianHandler) artifactCacheFilename(repository, path string) string {
246+
if repository == "" {
247+
return path
248+
}
249+
return repository + "/" + path
250+
}
251+
252+
// legacyArtifactCacheFilename is the name packages were cached under before
253+
// they were keyed by path: the file name, scoped by repository for a named
254+
// one. It is still read for the URL an entry was fetched from.
255+
func (h *DebianHandler) legacyArtifactCacheFilename(repository, filename string) string {
240256
if repository == "" {
241257
return filename
242258
}

‎internal/handler/debian_test.go‎

Lines changed: 78 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package handler
22

33
import (
4+
"context"
45
"fmt"
56
"net/http"
67
"net/http/httptest"
78
"strings"
9+
"sync/atomic"
810
"testing"
911
"time"
1012

@@ -37,10 +39,11 @@ func TestDebianHandler_Routes(t *testing.T) {
3739
}
3840

3941
// TestDebianHandler_LegacyCacheKeysUnchanged pins the artifact cache identity
40-
// the main archive used before named repositories existed. Deployments carry
41-
// warm caches across upgrades, so a changed key here silently discards them.
42-
// Metadata is cheap to refetch; its keys are covered by
43-
// TestMetadataCacheKeysUnambiguous.
42+
// the main archive used before named repositories existed. Packages are now
43+
// keyed by path, but deployments carry warm caches across upgrades, so
44+
// entries under this name are still read for the URL they were fetched from
45+
// (TestDebianHandler_LegacyPackageEntryServedForItsPath). Metadata is cheap
46+
// to refetch; its keys are covered by TestMetadataCacheKeysUnambiguous.
4447
func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) {
4548
const (
4649
poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb"
@@ -53,12 +56,12 @@ func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) {
5356

5457
// Artifact cache: the bare filename, with no repository prefix.
5558
filename := poolPath[strings.LastIndex(poolPath, "/")+1:]
56-
if got := h.artifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" {
59+
if got := h.legacyArtifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" {
5760
t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb")
5861
}
5962

6063
// A named repository must not reuse either identity.
61-
if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" {
64+
if got := h.legacyArtifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" {
6265
t.Error("named repository reused the legacy artifact cache filename")
6366
}
6467
if h.metadataCacheKeyFor("security", "https://security.test", distPath) == h.metadataCacheKeyFor("", h.upstreamURL, distPath) {
@@ -324,3 +327,72 @@ func TestDebRetentionKey(t *testing.T) {
324327
t.Error("debRetentionKey(nil) accepted")
325328
}
326329
}
330+
331+
// debianSuiteArchive serves a different build of one package file name per
332+
// suite, as Docker's repository nests a pool under each suite.
333+
func debianSuiteArchive(t *testing.T, available *atomic.Bool) *httptest.Server {
334+
t.Helper()
335+
archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
336+
if !available.Load() {
337+
http.Error(w, "unavailable", http.StatusServiceUnavailable)
338+
return
339+
}
340+
suite := strings.Split(strings.TrimPrefix(r.URL.Path, "/dists/"), "/")[0]
341+
_, _ = fmt.Fprint(w, suite+" build")
342+
}))
343+
t.Cleanup(archive.Close)
344+
return archive
345+
}
346+
347+
const (
348+
debianBullseyePackage = "/docker/dists/bullseye/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb"
349+
debianBookwormPackage = "/docker/dists/bookworm/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb"
350+
)
351+
352+
// TestDebianHandler_PackageCacheScopedPerPath checks that one file name in
353+
// two directories of a repository does not resolve to one cached artifact.
354+
func TestDebianHandler_PackageCacheScopedPerPath(t *testing.T) {
355+
var available atomic.Bool
356+
available.Store(true)
357+
archive := debianSuiteArchive(t, &available)
358+
proxy, _, _, _ := setupTestProxy(t)
359+
proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0))
360+
h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL})
361+
362+
for _, tc := range []struct{ path, want string }{
363+
{debianBullseyePackage, "bullseye build"},
364+
{debianBookwormPackage, "bookworm build"},
365+
} {
366+
if w := serveDebianRequest(h, tc.path); w.Code != http.StatusOK || w.Body.String() != tc.want {
367+
t.Errorf("GET %s = %d %q, want %q", tc.path, w.Code, w.Body.String(), tc.want)
368+
}
369+
}
370+
}
371+
372+
// TestDebianHandler_LegacyPackageEntryServedForItsPath checks that a package
373+
// cached under the file name alone, as before packages were keyed by path, is
374+
// still served for the path it was fetched from, and only for that path.
375+
func TestDebianHandler_LegacyPackageEntryServedForItsPath(t *testing.T) {
376+
var available atomic.Bool
377+
available.Store(true)
378+
archive := debianSuiteArchive(t, &available)
379+
proxy, _, _, _ := setupTestProxy(t)
380+
proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0))
381+
h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL})
382+
383+
// The entry an earlier version cached for the bullseye path.
384+
legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "deb", "containerd.io", "1.7.25-1",
385+
"docker/containerd.io_1.7.25-1_amd64.deb", archive.URL+strings.TrimPrefix(debianBullseyePackage, "/docker"))
386+
if err != nil {
387+
t.Fatalf("seed legacy entry: %v", err)
388+
}
389+
_ = legacy.Reader.Close()
390+
391+
if w := serveDebianRequest(h, debianBookwormPackage); w.Body.String() != "bookworm build" {
392+
t.Errorf("bookworm = %d %q, want its own build, not the legacy bullseye entry", w.Code, w.Body.String())
393+
}
394+
available.Store(false)
395+
if w := serveDebianRequest(h, debianBullseyePackage); w.Code != http.StatusOK || w.Body.String() != "bullseye build" {
396+
t.Errorf("bullseye with the archive down = %d %q, want the legacy entry", w.Code, w.Body.String())
397+
}
398+
}

0 commit comments

Comments
 (0)