Skip to content

Commit e09844e

Browse files
authored
Merge pull request #610 from simonchrz/feat/mirror-brew
Mirror Homebrew bottles
2 parents 54ad028 + 37ecc3d commit e09844e

10 files changed

Lines changed: 348 additions & 16 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ export HOMEBREW_ARTIFACT_DOMAIN=http://localhost:8080
261261

262262
The artifact domain proxies manifests and bottle blobs under `/v2/homebrew/core/`. GHCR routing is limited to that repository. Source archives, cask application downloads, custom tap artifacts, and legacy flat-file bottle mirrors use Homebrew's normal fallback URLs. Keep fallback enabled by leaving `HOMEBREW_ARTIFACT_DOMAIN_NO_FALLBACK` unset.
263263

264-
Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting.
264+
Enable `cache_metadata` or set `PROXY_CACHE_METADATA=true` to retain Homebrew JSON API responses for offline fallback. Bottle blobs and their OCI manifests are cached without this setting. `proxy mirror pkg:brew/<formula>` pre-fetches a formula's bottles (see [Homebrew bottles](docs/configuration.md#homebrew-bottles)).
265265

266266
The upstreams default to `https://formulae.brew.sh/api` for the JSON API and `https://ghcr.io` for artifacts. To chain this proxy to another proxy, configure its Homebrew endpoints as the upstreams:
267267

‎cmd/proxy/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,7 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.
619619
db *database.DB, store storage.Storage, logger *slog.Logger, workers int) *mirror.Mirror {
620620
m := mirror.New(proxy, db, store, logger, workers)
621621
m.Images = images
622+
m.Bottles = handler.NewHomebrewHandler(proxy, cfg.Upstream.HomebrewAPI)
622623
m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm)
623624
server.UseMirrorUpstreams(m, cfg.Upstream, proxy.HTTPClient)
624625
if cfg.Scanning.Enabled {

‎docs/configuration.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1670,6 +1670,21 @@ proxy mirror 'pkg:oci/debian@sha256:...?repository_url=docker.io/library/debian&
16701670
- Non-distributable (foreign) layers are skipped.
16711671
- While `scanning.enabled` is set, the command refuses images, because scanners fetch staged layers from the running server. Use the mirror API instead.
16721672

1673+
### Homebrew bottles
1674+
1675+
`pkg:brew` PURLs mirror a homebrew/core formula's bottles, which Homebrew publishes as an image on `upstream.homebrew_artifact`. They are mirrored like that image, under the keys brew reads through `HOMEBREW_ARTIFACT_DOMAIN`, so `brew install` is served from cache, also when the upstream is unreachable:
1676+
1677+
```bash
1678+
proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
1679+
```
1680+
1681+
- A formula name with `@` is written with `%40`, since `@` starts the version: `pkg:brew/openssl%403` is `openssl@3`, and `pkg:brew/openssl@3` is version 3 of `openssl`.
1682+
- The formula's JSON API on `upstream.homebrew_api` gives the image tag for the current version, which carries the formula's revision and the bottles' rebuild (`hello` 2.12.3 is tagged `2.12.3-1`). Only the current version is described there; another version is used as the tag as given, so give the full tag, such as `2.12.2` or `2.12.3-1`, for older bottles.
1683+
- A PURL without a version mirrors the current version.
1684+
- `--platform` selects bottles like image platforms: `linux/arm64`, `linux/amd64`, or `darwin/arm64` for every macOS release's arm64 bottle. Without it every bottle of the version is mirrored.
1685+
- Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`.
1686+
- Casks, source archives and taps other than homebrew/core aren't mirrored.
1687+
16731688
### Scheduled refresh
16741689

16751690
Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later:
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package handler
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"io"
9+
"net/http"
10+
"regexp"
11+
"strconv"
12+
"strings"
13+
)
14+
15+
var (
16+
homebrewFormulaPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9@+._-]*$`)
17+
homebrewVersionPattern = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z._+-]*$`)
18+
)
19+
20+
// homebrewMaxFormulaJSON bounds one formula's JSON API document, a few KB.
21+
const homebrewMaxFormulaJSON = 1 << 20
22+
23+
// homebrewFormula is the part of a formula's JSON API document that names
24+
// its bottles' image tag.
25+
type homebrewFormula struct {
26+
Versions struct {
27+
Stable string `json:"stable"`
28+
} `json:"versions"`
29+
Revision int `json:"revision"`
30+
Bottle struct {
31+
Stable struct {
32+
Rebuild int `json:"rebuild"`
33+
Files map[string]any `json:"files"`
34+
} `json:"stable"`
35+
} `json:"bottle"`
36+
}
37+
38+
// BottleImage returns the homebrew/core image and tag for a formula version.
39+
// Only the current version gets its revision and rebuild from the JSON API;
40+
// any other version is used as the tag as given.
41+
func (h *HomebrewHandler) BottleImage(ctx context.Context, formula, version string) (ImageRef, error) {
42+
if !homebrewFormulaPattern.MatchString(formula) {
43+
return ImageRef{}, fmt.Errorf("invalid Homebrew formula name %q", formula)
44+
}
45+
if !homebrewVersionPattern.MatchString(version) {
46+
return ImageRef{}, fmt.Errorf("invalid Homebrew version %q", version)
47+
}
48+
f, err := h.fetchFormula(ctx, formula)
49+
if err != nil {
50+
return ImageRef{}, err
51+
}
52+
tag := version
53+
if version == f.Versions.Stable {
54+
if len(f.Bottle.Stable.Files) == 0 {
55+
return ImageRef{}, fmt.Errorf("homebrew formula %s %s has no bottles", formula, version)
56+
}
57+
if f.Revision > 0 {
58+
tag += "_" + strconv.Itoa(f.Revision)
59+
}
60+
if f.Bottle.Stable.Rebuild > 0 {
61+
tag += "-" + strconv.Itoa(f.Bottle.Stable.Rebuild)
62+
}
63+
}
64+
name := strings.NewReplacer("@", "/", "+", "x").Replace(formula)
65+
return ImageRef{Name: homebrewArtifactRepository + "/" + name, References: []string{tag}}, nil
66+
}
67+
68+
func (h *HomebrewHandler) fetchFormula(ctx context.Context, formula string) (*homebrewFormula, error) {
69+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.apiUpstream+"/formula/"+formula+".json", nil)
70+
if err != nil {
71+
return nil, err
72+
}
73+
resp, err := h.proxy.HTTPClient.Do(req)
74+
if err != nil {
75+
return nil, fmt.Errorf("reading the Homebrew formula: %w", err)
76+
}
77+
defer func() { _ = resp.Body.Close() }()
78+
switch {
79+
case resp.StatusCode == http.StatusNotFound:
80+
return nil, fmt.Errorf("homebrew/core has no formula %q", formula)
81+
case resp.StatusCode != http.StatusOK:
82+
return nil, fmt.Errorf("reading the Homebrew formula: upstream answered %d", resp.StatusCode)
83+
}
84+
var f homebrewFormula
85+
if err := json.NewDecoder(io.LimitReader(resp.Body, homebrewMaxFormulaJSON)).Decode(&f); err != nil {
86+
return nil, fmt.Errorf("reading the Homebrew formula: %w", err)
87+
}
88+
if f.Versions.Stable == "" {
89+
return nil, errors.New("reading the Homebrew formula: it has no stable version")
90+
}
91+
return &f, nil
92+
}
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package handler
2+
3+
import (
4+
"context"
5+
"net/http"
6+
"net/http/httptest"
7+
"strings"
8+
"sync/atomic"
9+
"testing"
10+
)
11+
12+
func TestHomebrewBottleImage(t *testing.T) {
13+
formulas := map[string]string{
14+
"/formula/hello.json": `{"versions":{"stable":"2.12.3"},"revision":0,
15+
"bottle":{"stable":{"rebuild":1,"files":{"arm64_linux":{}}}}}`,
16+
"/formula/openssl@3.json": `{"versions":{"stable":"3.5.0"},"revision":2,
17+
"bottle":{"stable":{"rebuild":0,"files":{"arm64_sequoia":{}}}}}`,
18+
"/formula/libsigc++.json": `{"versions":{"stable":"3.6.0"},"revision":0,
19+
"bottle":{"stable":{"rebuild":0,"files":{"x86_64_linux":{}}}}}`,
20+
"/formula/nobottle.json": `{"versions":{"stable":"1.0"},"bottle":{}}`,
21+
}
22+
var requests atomic.Int32
23+
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
24+
requests.Add(1)
25+
body, ok := formulas[r.URL.Path]
26+
if !ok {
27+
http.NotFound(w, r)
28+
return
29+
}
30+
_, _ = w.Write([]byte(body))
31+
}))
32+
defer api.Close()
33+
proxy, _, _, _ := setupTestProxy(t)
34+
proxy.HTTPClient = api.Client()
35+
h := NewHomebrewHandler(proxy, api.URL)
36+
37+
tests := []struct {
38+
formula, version string
39+
image, tag, err string
40+
}{
41+
{formula: "hello", version: "2.12.3", image: "homebrew/core/hello", tag: "2.12.3-1"},
42+
{formula: "openssl@3", version: "3.5.0", image: "homebrew/core/openssl/3", tag: "3.5.0_2"},
43+
{formula: "libsigc++", version: "3.6.0", image: "homebrew/core/libsigcxx", tag: "3.6.0"},
44+
// Only the current version is described; another is the tag as given.
45+
{formula: "hello", version: "2.12.2", image: "homebrew/core/hello", tag: "2.12.2"},
46+
{formula: "hello", version: "2.12.3-1", image: "homebrew/core/hello", tag: "2.12.3-1"},
47+
{formula: "nobottle", version: "1.0", err: "has no bottles"},
48+
{formula: "missing", version: "1.0", err: `no formula "missing"`},
49+
}
50+
for _, tt := range tests {
51+
ref, err := h.BottleImage(context.Background(), tt.formula, tt.version)
52+
if tt.err != "" {
53+
if err == nil || !strings.Contains(err.Error(), tt.err) {
54+
t.Errorf("%s@%s: err = %v, want %q", tt.formula, tt.version, err, tt.err)
55+
}
56+
continue
57+
}
58+
if err != nil || ref.Name != tt.image || len(ref.References) != 1 || ref.References[0] != tt.tag {
59+
t.Errorf("%s@%s = %+v, %v; want %s:%s", tt.formula, tt.version, ref, err, tt.image, tt.tag)
60+
}
61+
}
62+
63+
before := requests.Load()
64+
for _, bad := range [][2]string{{"../hello", "1.0"}, {"Hello", "1.0"}, {"hello", "1.0/../x"}, {"hello", ""}} {
65+
if _, err := h.BottleImage(context.Background(), bad[0], bad[1]); err == nil {
66+
t.Errorf("%q@%q accepted", bad[0], bad[1])
67+
}
68+
}
69+
if requests.Load() != before {
70+
t.Error("an invalid name or version reached the API")
71+
}
72+
}

‎internal/mirror/brew_test.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package mirror
2+
3+
import (
4+
"context"
5+
"errors"
6+
"strings"
7+
"sync"
8+
"testing"
9+
10+
"github.com/git-pkgs/proxy/internal/handler"
11+
)
12+
13+
// fakeBottles resolves every formula version to homebrew/core/<name> tagged
14+
// <version>-1, like a formula whose bottles were rebuilt once.
15+
type fakeBottles struct {
16+
err error
17+
}
18+
19+
func (f fakeBottles) BottleImage(_ context.Context, formula, version string) (handler.ImageRef, error) {
20+
if f.err != nil {
21+
return handler.ImageRef{}, f.err
22+
}
23+
return handler.ImageRef{Name: "homebrew/core/" + formula, References: []string{version + "-1"}}, nil
24+
}
25+
26+
// platformImages records the images mirrored and the platforms asked for.
27+
type platformImages struct {
28+
mu sync.Mutex
29+
refs []handler.ImageRef
30+
platforms [][]handler.Platform
31+
}
32+
33+
func (p *platformImages) MirrorImage(_ context.Context, ref handler.ImageRef, platforms []handler.Platform) (handler.ImageResult, error) {
34+
p.mu.Lock()
35+
defer p.mu.Unlock()
36+
p.refs = append(p.refs, ref)
37+
p.platforms = append(p.platforms, platforms)
38+
return handler.ImageResult{Fetched: 2, Bytes: 10}, nil
39+
}
40+
41+
type stubVersions map[string][]string
42+
43+
func (s stubVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) {
44+
return s[ecosystem+"/"+name], nil
45+
}
46+
47+
func TestMirrorBrewMirrorsBottleImage(t *testing.T) {
48+
m := setupTestMirror(t, 1)
49+
images := &platformImages{}
50+
m.Images, m.Bottles = images, fakeBottles{}
51+
52+
source := &PURLSource{
53+
PURLs: []string{"pkg:brew/hello@2.12.3", "pkg:brew/jq"},
54+
Platforms: []string{"linux/arm64"},
55+
Versions: stubVersions{"brew/jq": {"1.8.1"}},
56+
}
57+
progress, err := m.Run(context.Background(), source)
58+
if err != nil {
59+
t.Fatal(err)
60+
}
61+
if progress.Completed != 2 || progress.Failed != 0 {
62+
t.Fatalf("progress = %+v, errors %v", progress, progress.Errors)
63+
}
64+
got := map[string]string{}
65+
for i, ref := range images.refs {
66+
if len(images.platforms[i]) != 1 || images.platforms[i][0].OS != "linux" || images.platforms[i][0].Architecture != "arm64" {
67+
t.Errorf("%s: platforms = %+v, want linux/arm64", ref.Name, images.platforms[i])
68+
}
69+
got[ref.Name] = strings.Join(ref.References, ",")
70+
}
71+
if got["homebrew/core/hello"] != "2.12.3-1" || got["homebrew/core/jq"] != "1.8.1-1" {
72+
t.Errorf("images = %v", got)
73+
}
74+
}
75+
76+
func TestMirrorBrewFailures(t *testing.T) {
77+
m := setupTestMirror(t, 1)
78+
progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/hello@2.12.3"}})
79+
if err != nil {
80+
t.Fatal(err)
81+
}
82+
if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "isn't available") {
83+
t.Errorf("without a bottle resolver: %+v", progress)
84+
}
85+
86+
images := &platformImages{}
87+
m.Images, m.Bottles = images, fakeBottles{err: errors.New(`homebrew/core has no formula "nope"`)}
88+
progress, err = m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:brew/nope@1.0"}})
89+
if err != nil {
90+
t.Fatal(err)
91+
}
92+
if progress.Failed != 1 || len(images.refs) != 0 {
93+
t.Errorf("unknown formula: %+v, mirrored %v", progress, images.refs)
94+
}
95+
}

‎internal/mirror/image.go‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,18 @@ func isImageEcosystem(ecosystem string) bool {
2727
return ecosystem == "docker" || ecosystem == "oci"
2828
}
2929

30+
// BottleResolver resolves a Homebrew formula version to the image its
31+
// bottles are published under.
32+
type BottleResolver interface {
33+
BottleImage(ctx context.Context, formula, version string) (handler.ImageRef, error)
34+
}
35+
36+
// takesPlatforms reports whether --platform applies to an ecosystem: images,
37+
// and Homebrew bottles, which are published as images.
38+
func takesPlatforms(ecosystem string) bool {
39+
return isImageEcosystem(ecosystem) || ecosystem == "brew"
40+
}
41+
3042
// ParsePlatforms parses os/architecture[/variant] values.
3143
func ParsePlatforms(values []string) ([]handler.Platform, error) {
3244
var platforms []handler.Platform
@@ -98,6 +110,29 @@ func imageRef(pv PackageVersion) (handler.ImageRef, []handler.Platform, error) {
98110
}
99111

100112
func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
113+
ref, platforms, err := imageRef(pv)
114+
m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker)
115+
}
116+
117+
// mirrorBrew mirrors the bottles of a Homebrew formula version, which brew
118+
// pulls as an image from homebrew/core on its artifact domain.
119+
func (m *Mirror) mirrorBrew(ctx context.Context, pv PackageVersion, tracker *progressTracker) {
120+
if m.Images == nil || m.Bottles == nil {
121+
m.recordFailure(tracker, pv, errors.New("mirroring Homebrew bottles isn't available here"))
122+
return
123+
}
124+
platforms, err := ParsePlatforms(pv.Platforms)
125+
if err != nil {
126+
m.recordFailure(tracker, pv, err)
127+
return
128+
}
129+
ref, err := m.Bottles.BottleImage(ctx, pv.Name, pv.Version)
130+
m.mirrorImageRef(ctx, pv, ref, platforms, err, tracker)
131+
}
132+
133+
// mirrorImageRef mirrors the image ref for pv, or records refErr, the error
134+
// resolving it.
135+
func (m *Mirror) mirrorImageRef(ctx context.Context, pv PackageVersion, ref handler.ImageRef, platforms []handler.Platform, refErr error, tracker *progressTracker) {
101136
fail := func(err error) {
102137
tracker.failed.Add(1)
103138
tracker.addError(pv.Ecosystem, pv.Name, pv.Version, err.Error())
@@ -107,9 +142,8 @@ func (m *Mirror) mirrorImage(ctx context.Context, pv PackageVersion, tracker *pr
107142
fail(fmt.Errorf("%w: %s", handler.ErrVersionDenied, pv))
108143
return
109144
}
110-
ref, platforms, err := imageRef(pv)
111-
if err != nil {
112-
fail(err)
145+
if refErr != nil {
146+
fail(refErr)
113147
return
114148
}
115149
result, err := m.Images.MirrorImage(ctx, ref, platforms)

‎internal/mirror/mirror.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ type Mirror struct {
3232
// Charts mirrors helm charts. Without it they fail.
3333
Charts ChartMirrorer
3434

35+
// Bottles resolves Homebrew formula versions to the images their
36+
// bottles are published under, which Images mirrors. Without it brew
37+
// PURLs fail.
38+
Bottles BottleResolver
39+
3540
// Versions lists the versions of unversioned PURLs in mirror jobs and
3641
// schedules. Nil lists them from the public registries.
3742
Versions VersionLister
@@ -228,6 +233,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
228233
m.mirrorImage(ctx, pv, tracker)
229234
return
230235
}
236+
if pv.Ecosystem == "brew" {
237+
m.mirrorBrew(ctx, pv, tracker)
238+
return
239+
}
231240
if m.RefusePackages != "" {
232241
tracker.failed.Add(1)
233242
tracker.addError(pv.Ecosystem, pv.Name, pv.Version, m.RefusePackages)

0 commit comments

Comments
 (0)