Skip to content

Commit 72aa8fd

Browse files
authored
Merge pull request #667 from simonchrz/feat/mirror-git
Mirror pkg:git commits into the git cache
2 parents 7e18c6a + b0566aa commit 72aa8fd

18 files changed

Lines changed: 934 additions & 35 deletions

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1209,6 +1209,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
12091209
12101210
# Mirror a container image for one platform
12111211
proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
1212+
1213+
# Mirror a commit of an upstream.git repository into the git cache
1214+
proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
12121215
```
12131216

12141217
The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror.

‎cmd/proxy/main.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,9 @@ func runMirror() {
595595
return
596596
}
597597

598+
closeGit := useCommandGit(cfg, proxy, m, logger)
598599
progress, err := m.Run(ctx, source)
600+
closeGit()
599601
if err != nil {
600602
fmt.Fprintf(os.Stderr, "error: %v\n", err)
601603
os.Exit(1)
@@ -634,6 +636,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.
634636
return m
635637
}
636638

639+
// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
640+
// when upstream.git is configured, and returns the function that closes it.
641+
// Where it can't be opened, such as while the running server holds
642+
// git.cache_dir, those items fail with the reason and the rest are mirrored.
643+
func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
644+
git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
645+
if err != nil {
646+
m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
647+
return func() {}
648+
}
649+
if git == nil {
650+
return func() {}
651+
}
652+
m.Git = git
653+
return func() {
654+
if err := closeGit(); err != nil {
655+
logger.Warn("closing the git cache", "error", err)
656+
}
657+
}
658+
}
659+
637660
// mirrorExitCode returns the exit status for a finished mirror run: 1 when any
638661
// package version failed to mirror, 0 otherwise.
639662
func mirrorExitCode(progress *mirror.Progress) int {

‎cmd/proxy/mirror_git_test.go‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package main
2+
3+
import (
4+
"net/http"
5+
"net/http/cgi"
6+
"net/http/httptest"
7+
"os"
8+
"os/exec"
9+
"path/filepath"
10+
"strings"
11+
"testing"
12+
"time"
13+
14+
"github.com/git-pkgs/proxy/internal/gitcache"
15+
)
16+
17+
// gitMirrorUpstream serves team/repo, with one commit on main, from git
18+
// http-backend, and returns the server and the commit.
19+
func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
20+
t.Helper()
21+
git := func(dir string, args ...string) string {
22+
t.Helper()
23+
cmd := exec.Command("git", args...)
24+
cmd.Dir = dir
25+
cmd.Env = []string{
26+
"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
27+
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
28+
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
29+
}
30+
out, err := cmd.CombinedOutput()
31+
if err != nil {
32+
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
33+
}
34+
return strings.TrimSpace(string(out))
35+
}
36+
backend := ""
37+
for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
38+
if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
39+
backend = path
40+
}
41+
}
42+
if backend == "" {
43+
t.Skip("git http-backend is not installed")
44+
}
45+
root := t.TempDir()
46+
bare := filepath.Join(root, "team", "repo.git")
47+
git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
48+
work := t.TempDir()
49+
git("", "clone", "--quiet", bare, work)
50+
git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
51+
git(work, "push", "--quiet", "origin", "HEAD:main")
52+
head := git(work, "rev-parse", "HEAD")
53+
54+
server := httptest.NewServer(&cgi.Handler{
55+
Path: backend,
56+
Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
57+
})
58+
t.Cleanup(server.Close)
59+
return server, head
60+
}
61+
62+
func fileExistsForTest(path string) bool {
63+
_, err := os.Stat(path)
64+
return err == nil
65+
}
66+
67+
// writeGitMirrorConfig writes a config with upstream.git "up" at
68+
// upstreamURL and the git cache, storage and database under dir.
69+
func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
70+
t.Helper()
71+
cfg := "storage:\n" +
72+
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
73+
"database:\n" +
74+
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
75+
"upstream:\n" +
76+
" allow_loopback: true\n" +
77+
" git:\n" +
78+
" up: " + upstreamURL + "\n" +
79+
"git:\n" +
80+
" cache_dir: " + filepath.Join(dir, "git") + "\n"
81+
path := filepath.Join(dir, "config.yaml")
82+
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
83+
t.Fatalf("writing config: %v", err)
84+
}
85+
return path
86+
}
87+
88+
func TestMirrorCommandCachesGitCommit(t *testing.T) {
89+
upstream, head := gitMirrorUpstream(t)
90+
dir := t.TempDir()
91+
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
92+
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
93+
94+
output := runMirrorProcess(t, 0, "-config", configPath, purl)
95+
if !strings.Contains(output, "1 downloaded") {
96+
t.Errorf("output:\n%s", output)
97+
}
98+
cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
99+
if len(cached) != 1 {
100+
t.Errorf("cached repositories %v, want team/repo.git", cached)
101+
}
102+
103+
output = runMirrorProcess(t, 0, "-config", configPath, purl)
104+
if !strings.Contains(output, "1 skipped") {
105+
t.Errorf("second run output:\n%s", output)
106+
}
107+
}
108+
109+
// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
110+
// mirror API, while the running server holds git.cache_dir.
111+
func TestMirrorCommandGitCacheInUse(t *testing.T) {
112+
upstream, head := gitMirrorUpstream(t)
113+
dir := t.TempDir()
114+
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
115+
cache, err := gitcache.New(gitcache.Options{
116+
CacheDir: filepath.Join(dir, "git"),
117+
RefreshWait: time.Second,
118+
Client: &http.Client{},
119+
})
120+
if err != nil {
121+
t.Fatal(err)
122+
}
123+
defer func() { _ = cache.Close() }()
124+
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
125+
126+
output := runMirrorProcess(t, 1, "-config", configPath, purl)
127+
for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
128+
if !strings.Contains(output, want) {
129+
t.Errorf("output missing %q:\n%s", want, output)
130+
}
131+
}
132+
}

‎docs/architecture.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g
363363

364364
### `internal/mirror`
365365

366-
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads.
366+
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves.
367367

368368
The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
369369

‎docs/configuration.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1908,6 +1908,20 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
19081908
- Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`.
19091909
- Casks, source archives and taps other than homebrew/core aren't mirrored.
19101910

1911+
### Git repositories
1912+
1913+
`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID:
1914+
1915+
```bash
1916+
proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
1917+
proxy mirror pkg:git/github.com/git-pkgs/purl
1918+
```
1919+
1920+
- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail.
1921+
- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails.
1922+
- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID.
1923+
- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead.
1924+
19111925
### Scheduled refresh
19121926

19131927
Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later:

‎internal/gitcache/mirror.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package gitcache
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"strings"
8+
"time"
9+
)
10+
11+
// MirrorResult is what Mirror did.
12+
type MirrorResult struct {
13+
// Cached reports that the repository, and the commit if one was given,
14+
// were cached already.
15+
Cached bool
16+
// Bytes is how much the cached repository grew.
17+
Bytes int64
18+
}
19+
20+
// Mirror brings the repository cached under key up to date with repoURL,
21+
// cloning it if needed, and waits for that to finish. With commit, an object
22+
// ID, it also makes sure the commit is cached. One on no branch or tag is
23+
// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and
24+
// is kept as long as a commit a client fetched by ID.
25+
func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) {
26+
if commit != "" && !isObjectID(commit) {
27+
return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit)
28+
}
29+
lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants})
30+
if err != nil {
31+
return MirrorResult{}, err
32+
}
33+
defer lease.Release()
34+
rp := lease.repo
35+
cached := lease.hit
36+
before := int64(0)
37+
var refreshErr error
38+
// Open cloned a repository that wasn't cached; one that was is
39+
// refreshed here, waiting for the fetch a client would not wait for.
40+
if cached {
41+
before = rp.size.Load()
42+
refresh := c.startRefresh(rp, repoURL)
43+
select {
44+
case <-refresh.done:
45+
refreshErr = refresh.err
46+
case <-ctx.Done():
47+
return MirrorResult{}, ctx.Err()
48+
}
49+
}
50+
51+
// A refresh past max_repo_size gets the repository dropped once it is
52+
// released, commit and all.
53+
if errors.Is(refreshErr, ErrTooLarge) {
54+
return MirrorResult{}, refreshErr
55+
}
56+
if commit != "" {
57+
if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil {
58+
return MirrorResult{}, err
59+
}
60+
}
61+
// A refresh that failed leaves the commit cached all the same.
62+
if refreshErr != nil && commit == "" {
63+
return MirrorResult{}, refreshErr
64+
}
65+
grown := max(rp.size.Load()-before, 0)
66+
return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil
67+
}
68+
69+
// ensureCommit makes sure the repository has commit, fetching it by ID when
70+
// it doesn't and fetchWants allows it. A commit on no branch or tag that is
71+
// cached already gets a new wanted ref, so it is kept for wantedMaxAge from
72+
// this run on.
73+
func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error {
74+
missing, err := c.missingObjects(ctx, rp.dir, []string{commit})
75+
if err != nil {
76+
return err
77+
}
78+
if len(missing) == 0 {
79+
return c.renewWanted(ctx, rp, commit)
80+
}
81+
if !fetchWants {
82+
return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
83+
}
84+
if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil {
85+
if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) {
86+
return err
87+
}
88+
// The upstream refuses a commit it doesn't have.
89+
return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err)
90+
}
91+
// fetchMissingWants skips a commit that failed moments ago.
92+
if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil {
93+
return err
94+
}
95+
if len(missing) > 0 {
96+
return fmt.Errorf("commit %s: %w", commit, ErrNotFound)
97+
}
98+
return nil
99+
}
100+
101+
// renewWanted writes a new wanted ref for commit when no branch or tag
102+
// includes it.
103+
func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error {
104+
unlock, err := rp.lockWrites(ctx)
105+
if err != nil {
106+
return err
107+
}
108+
defer unlock()
109+
unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit})
110+
if err != nil || !unmerged[commit] {
111+
return err
112+
}
113+
cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit)
114+
if out, err := cmd.CombinedOutput(); err != nil {
115+
return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out)))
116+
}
117+
return nil
118+
}

0 commit comments

Comments
 (0)