Skip to content

Commit d950919

Browse files
authored
Use shared artifacts at cache boundaries (#267)
* Use shared artifacts at cache boundaries * Address artifact cache review * Defer cached artifact validation to checkCache Construct artifacts.Artifact from the row without validation so a malformed content hash reaches newIntegrityChecks in checkCache, which clears the row and treats the request as a miss. Erroring at the DB boundary instead surfaced a 500 and left the bad row in place. * Build stored Artifact after digest and scan checks Construct the shared artifact struct from trusted storage output as a literal, after the hash-mismatch and scanner paths that delete failed downloads, so no path between Store and updateCacheDB can leave bytes in storage without a database row.
1 parent 230fa7b commit d950919

14 files changed

Lines changed: 222 additions & 85 deletions

File tree

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/aws/aws-sdk-go-v2/config v1.32.40
99
github.com/aws/aws-sdk-go-v2/service/ecr v1.61.0
1010
github.com/git-pkgs/archives v0.5.1
11+
github.com/git-pkgs/artifacts v0.2.0
1112
github.com/git-pkgs/cooldown v0.2.0
1213
github.com/git-pkgs/enrichment v0.7.0
1314
github.com/git-pkgs/gcs v0.1.0
@@ -21,6 +22,7 @@ require (
2122
github.com/go-chi/chi/v5 v5.3.2
2223
github.com/jmoiron/sqlx v1.4.0
2324
github.com/lib/pq v1.12.3
25+
github.com/opencontainers/go-digest v1.0.0
2426
github.com/prometheus/client_golang v1.24.1
2527
github.com/prometheus/client_model v0.6.2
2628
github.com/spdx/tools-golang v0.5.7
@@ -132,7 +134,6 @@ require (
132134
github.com/fsnotify/fsnotify v1.9.0 // indirect
133135
github.com/fzipp/gocyclo v0.6.0 // indirect
134136
github.com/ghostiam/protogetter v0.3.21 // indirect
135-
github.com/git-pkgs/artifacts v0.2.0 // indirect
136137
github.com/git-pkgs/packageurl-go v0.3.1 // indirect
137138
github.com/git-pkgs/pom v0.1.7 // indirect
138139
github.com/github/go-spdx/v2 v2.7.0 // indirect
@@ -228,7 +229,6 @@ require (
228229
github.com/nunnatsa/ginkgolinter v0.24.0 // indirect
229230
github.com/oapi-codegen/nullable v1.2.0 // indirect
230231
github.com/oapi-codegen/runtime v1.6.0 // indirect
231-
github.com/opencontainers/go-digest v1.0.0 // indirect
232232
github.com/package-url/packageurl-go v0.1.7 // indirect
233233
github.com/pandatix/go-cvss v0.6.2 // indirect
234234
github.com/pelletier/go-toml v1.9.5 // indirect

‎internal/database/database_test.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -279,14 +279,20 @@ func TestGetCachedArtifact(t *testing.T) {
279279
if cached.StoragePath != "/cache/npm/"+filename {
280280
t.Errorf("expected cached storage path, got %q", cached.StoragePath)
281281
}
282-
if cached.ContentHash.String != testContentHash {
283-
t.Errorf("expected cached content hash, got %q", cached.ContentHash.String)
282+
if cached.Artifact.PURL != versionPURL {
283+
t.Errorf("expected cached PURL %q, got %q", versionPURL, cached.Artifact.PURL)
284284
}
285-
if cached.Size.Int64 != 12345 {
286-
t.Errorf("expected cached size 12345, got %d", cached.Size.Int64)
285+
if cached.Artifact.Digest.String() != "sha256:"+testContentHash {
286+
t.Errorf("expected cached digest, got %q", cached.Artifact.Digest)
287287
}
288-
if cached.ContentType.String != "application/gzip" {
289-
t.Errorf("expected cached content type, got %q", cached.ContentType.String)
288+
if cached.Artifact.Size != 12345 {
289+
t.Errorf("expected cached size 12345, got %d", cached.Artifact.Size)
290+
}
291+
if cached.Artifact.Filename != filename {
292+
t.Errorf("expected cached filename %q, got %q", filename, cached.Artifact.Filename)
293+
}
294+
if cached.Artifact.MediaType != "application/gzip" {
295+
t.Errorf("expected cached content type, got %q", cached.Artifact.MediaType)
290296
}
291297
if cached.Integrity.String != testIntegrity {
292298
t.Errorf("expected cached integrity, got %q", cached.Integrity.String)

‎internal/database/queries.go‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ import (
44
"database/sql"
55
"fmt"
66
"time"
7+
8+
"github.com/git-pkgs/artifacts"
9+
"github.com/opencontainers/go-digest"
710
)
811

912
// Package queries
@@ -225,7 +228,7 @@ func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) {
225228

226229
// GetCachedArtifact returns the fields needed to serve a cached artifact.
227230
func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*CachedArtifact, error) {
228-
var artifact CachedArtifact
231+
var row cachedArtifactRow
229232
query := db.Rebind(`
230233
SELECT packages.ecosystem, artifacts.storage_path, artifacts.content_hash, artifacts.size,
231234
artifacts.content_type, versions.integrity
@@ -235,14 +238,42 @@ func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*Cac
235238
WHERE packages.purl = ? AND artifacts.version_purl = ? AND artifacts.filename = ?
236239
AND artifacts.storage_path IS NOT NULL AND artifacts.fetched_at IS NOT NULL
237240
`)
238-
err := db.Get(&artifact, query, packagePURL, versionPURL, filename)
241+
err := db.Get(&row, query, packagePURL, versionPURL, filename)
239242
if err == sql.ErrNoRows {
240243
return nil, nil
241244
}
242245
if err != nil {
243246
return nil, err
244247
}
245-
return &artifact, nil
248+
return row.artifact(versionPURL, filename), nil
249+
}
250+
251+
type cachedArtifactRow struct {
252+
Ecosystem string `db:"ecosystem"`
253+
StoragePath string `db:"storage_path"`
254+
ContentHash sql.NullString `db:"content_hash"`
255+
Size sql.NullInt64 `db:"size"`
256+
ContentType sql.NullString `db:"content_type"`
257+
Integrity sql.NullString `db:"integrity"`
258+
}
259+
260+
// artifact converts a cached artifact row to a CachedArtifact without
261+
// validation. A malformed hash or integrity value is handled by
262+
// checkCache, which clears the record and treats the request as a cache
263+
// miss so the client is served a fresh fetch instead of an error.
264+
func (row cachedArtifactRow) artifact(versionPURL, filename string) *CachedArtifact {
265+
return &CachedArtifact{
266+
Ecosystem: row.Ecosystem,
267+
StoragePath: row.StoragePath,
268+
Integrity: row.Integrity,
269+
Artifact: artifacts.Artifact{
270+
PURL: versionPURL,
271+
Digest: digest.Digest("sha256:" + row.ContentHash.String),
272+
Size: row.Size.Int64,
273+
Filename: filename,
274+
MediaType: row.ContentType.String,
275+
},
276+
}
246277
}
247278

248279
func (db *DB) GetArtifactByPath(storagePath string) (*Artifact, error) {

‎internal/database/types.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import (
55
"net/url"
66
"strings"
77
"time"
8+
9+
"github.com/git-pkgs/artifacts"
810
)
911

1012
// Package represents a package in the database.
@@ -148,12 +150,10 @@ func (a *Artifact) IsCached() bool {
148150

149151
// CachedArtifact contains the fields needed to serve a cached artifact.
150152
type CachedArtifact struct {
151-
Ecosystem string `db:"ecosystem"`
152-
StoragePath string `db:"storage_path"`
153-
ContentHash sql.NullString `db:"content_hash"`
154-
Size sql.NullInt64 `db:"size"`
155-
ContentType sql.NullString `db:"content_type"`
156-
Integrity sql.NullString `db:"integrity"`
153+
Ecosystem string
154+
StoragePath string
155+
Artifact artifacts.Artifact
156+
Integrity sql.NullString
157157
}
158158

159159
// MetadataCacheEntry represents a cached metadata blob for offline serving.

‎internal/handler/apk.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,8 +129,8 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques
129129
return
130130
}
131131

132-
if result.ContentType == "" {
133-
result.ContentType = "application/octet-stream"
132+
if result.Artifact.MediaType == "" {
133+
result.Artifact.MediaType = "application/octet-stream"
134134
}
135135
serveArtifact(w, r.Method, result)
136136
}

‎internal/handler/container.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,8 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req
161161
}
162162
if cached != nil {
163163
w.Header().Set("Docker-Content-Digest", digest)
164-
if cached.ContentType == "" {
165-
cached.ContentType = "application/octet-stream"
164+
if cached.Artifact.MediaType == "" {
165+
cached.Artifact.MediaType = "application/octet-stream"
166166
}
167167
serveArtifact(w, r.Method, cached)
168168
return
@@ -205,8 +205,8 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req
205205
}
206206

207207
w.Header().Set("Docker-Content-Digest", digest)
208-
if result.ContentType == "" {
209-
result.ContentType = "application/octet-stream"
208+
if result.Artifact.MediaType == "" {
209+
result.Artifact.MediaType = "application/octet-stream"
210210
}
211211
ServeArtifact(w, result)
212212
}

‎internal/handler/download_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,14 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos
4444

4545
storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename)
4646
store.files[storagePath] = []byte(content)
47+
sharedArtifact := testArtifact(content, versionPURL, filename, "application/octet-stream")
4748

4849
art := &database.Artifact{
4950
VersionPURL: versionPURL,
5051
Filename: filename,
5152
UpstreamURL: "https://example.com/" + filename,
5253
StoragePath: sql.NullString{String: storagePath, Valid: true},
53-
ContentHash: sql.NullString{String: sha256Hex(content), Valid: true},
54+
ContentHash: sql.NullString{String: sharedArtifact.Digest.Encoded(), Valid: true},
5455
Size: sql.NullInt64{Int64: int64(len(content)), Valid: true},
5556
ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
5657
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},

‎internal/handler/generic.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,8 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque
130130
return
131131
}
132132

133-
if result.ContentType == "" {
134-
result.ContentType = "application/octet-stream"
133+
if result.Artifact.MediaType == "" {
134+
result.Artifact.MediaType = "application/octet-stream"
135135
}
136136
serveArtifact(w, r.Method, result)
137137
}

‎internal/handler/handler.go‎

Lines changed: 38 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"sync"
1717
"time"
1818

19+
"github.com/git-pkgs/artifacts"
1920
"github.com/git-pkgs/cooldown"
2021
"github.com/git-pkgs/proxy/internal/database"
2122
"github.com/git-pkgs/proxy/internal/metrics"
@@ -24,6 +25,7 @@ import (
2425
"github.com/git-pkgs/proxy/internal/storage"
2526
"github.com/git-pkgs/purl"
2627
"github.com/git-pkgs/registries/fetch"
28+
"github.com/opencontainers/go-digest"
2729
)
2830

2931
// containsPathTraversal returns true if the path contains ".." segments
@@ -205,9 +207,7 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter
205207
type CacheResult struct {
206208
Reader io.ReadCloser
207209
RedirectURL string
208-
Size int64
209-
ContentType string
210-
Hash string
210+
Artifact artifacts.Artifact
211211
Cached bool
212212
storagePath string
213213
}
@@ -270,16 +270,14 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
270270
if artifact == nil {
271271
return nil, nil
272272
}
273-
checks, err := newIntegrityChecks(artifact.ContentHash.String, artifact.Integrity.String)
273+
checks, err := newIntegrityChecks(artifact.Artifact.Digest.Encoded(), artifact.Integrity.String)
274274
if err != nil {
275275
p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
276276
return nil, nil
277277
}
278278

279279
result := &CacheResult{
280-
Size: artifact.Size.Int64,
281-
ContentType: artifact.ContentType.String,
282-
Hash: artifact.ContentHash.String,
280+
Artifact: artifact.Artifact,
283281
Cached: true,
284282
storagePath: artifact.StoragePath,
285283
}
@@ -439,8 +437,16 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil
439437
}
440438
}
441439

440+
sharedArtifact := artifacts.Artifact{
441+
PURL: versionPURL,
442+
Digest: digest.Digest("sha256:" + hash),
443+
Size: size,
444+
Filename: filename,
445+
MediaType: artifact.ContentType,
446+
}
447+
442448
// Update database
443-
if err := p.updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, upstreamURL, storagePath, hash, size, artifact.ContentType); err != nil {
449+
if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact); err != nil {
444450
p.Logger.Warn("failed to update cache database", "error", err)
445451
// Continue anyway - we have the file
446452
}
@@ -456,11 +462,9 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil
456462
}
457463

458464
return &CacheResult{
459-
Reader: reader,
460-
Size: size,
461-
ContentType: artifact.ContentType,
462-
Hash: hash,
463-
Cached: false,
465+
Reader: reader,
466+
Artifact: sharedArtifact,
467+
Cached: false,
464468
}, nil
465469
}
466470

@@ -501,7 +505,7 @@ func (p *Proxy) runScan(ctx context.Context, ecosystem, name, version, filename,
501505
return nil
502506
}
503507

504-
func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, upstreamURL, storagePath, hash string, size int64, contentType string) error {
508+
func (p *Proxy) updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath string, artifact artifacts.Artifact) error {
505509
now := time.Now()
506510

507511
// Upsert package
@@ -518,7 +522,7 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u
518522

519523
// Upsert version
520524
ver := &database.Version{
521-
PURL: versionPURL,
525+
PURL: artifact.PURL,
522526
PackagePURL: pkgPURL,
523527
EnrichedAt: sql.NullTime{Time: now, Valid: true},
524528
}
@@ -528,13 +532,13 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u
528532

529533
// Upsert artifact
530534
art := &database.Artifact{
531-
VersionPURL: versionPURL,
532-
Filename: filename,
535+
VersionPURL: artifact.PURL,
536+
Filename: artifact.Filename,
533537
UpstreamURL: upstreamURL,
534538
StoragePath: sql.NullString{String: storagePath, Valid: true},
535-
ContentHash: sql.NullString{String: hash, Valid: true},
536-
Size: sql.NullInt64{Int64: size, Valid: true},
537-
ContentType: sql.NullString{String: contentType, Valid: true},
539+
ContentHash: sql.NullString{String: artifact.Digest.Encoded(), Valid: true},
540+
Size: sql.NullInt64{Int64: artifact.Size, Valid: true},
541+
ContentType: sql.NullString{String: artifact.MediaType, Valid: true},
538542
FetchedAt: sql.NullTime{Time: now, Valid: true},
539543
}
540544
if err := p.DB.UpsertArtifact(art); err != nil {
@@ -550,9 +554,13 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) {
550554
}
551555

552556
func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) {
557+
contentHash := ""
558+
if result.Artifact.Digest != "" {
559+
contentHash = result.Artifact.Digest.Encoded()
560+
}
553561
if result.RedirectURL != "" {
554-
if result.Hash != "" {
555-
w.Header().Set(headerETag, `"`+result.Hash+`"`)
562+
if contentHash != "" {
563+
w.Header().Set(headerETag, `"`+contentHash+`"`)
556564
}
557565
w.Header().Set("Location", result.RedirectURL)
558566
w.WriteHeader(http.StatusFound)
@@ -563,14 +571,14 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) {
563571
defer func() { _ = result.Reader.Close() }()
564572
}
565573

566-
if result.ContentType != "" {
567-
w.Header().Set(headerContentType, result.ContentType)
574+
if result.Artifact.MediaType != "" {
575+
w.Header().Set(headerContentType, result.Artifact.MediaType)
568576
}
569-
if result.Size > 0 || (method == http.MethodHead && result.Size == 0) {
570-
w.Header().Set(headerContentLength, strconv.FormatInt(result.Size, 10))
577+
if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) {
578+
w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10))
571579
}
572-
if result.Hash != "" {
573-
w.Header().Set(headerETag, `"`+result.Hash+`"`)
580+
if contentHash != "" {
581+
w.Header().Set(headerETag, `"`+contentHash+`"`)
574582
}
575583

576584
w.WriteHeader(http.StatusOK)
@@ -1107,15 +1115,15 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL,
11071115
if err != nil || cached == nil {
11081116
return cached, err
11091117
}
1110-
if artifactHashMatches(cached.Hash, upstreamHash) {
1118+
if artifactHashMatches(cached.Artifact.Digest.Encoded(), upstreamHash) {
11111119
return cached, nil
11121120
}
11131121

11141122
if cached.Reader != nil {
11151123
_ = cached.Reader.Close()
11161124
}
11171125
p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding",
1118-
"purl", versionPURL, "filename", filename, "cached", cached.Hash, "upstream", upstreamHash)
1126+
"purl", versionPURL, "filename", filename, "cached", cached.Artifact.Digest.Encoded(), "upstream", upstreamHash)
11191127
p.discardCachedArtifact(ctx, versionPURL, filename, cached.storagePath)
11201128
return nil, nil
11211129
}

0 commit comments

Comments
 (0)