Skip to content

Commit cf57402

Browse files
authored
Merge pull request #707 from simonchrz/feat/git-cache-delete
Delete a cached Git repository from the UI and API
2 parents db78664 + 4cd5e4f commit cf57402

13 files changed

Lines changed: 476 additions & 11 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1381,6 +1381,7 @@ Registered when `cache_delete` is set. It has no authentication of its own.
13811381
| Endpoint | Description |
13821382
|----------|-------------|
13831383
| `DELETE /api/cache/{ecosystem}/{name}/{version}` | Delete a version's cached files, or only `?filename=` |
1384+
| `DELETE /api/git/{upstream}/{repo}` | Delete a repository from the git cache |
13841385

13851386
### Enrichment API
13861387

‎config.example.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@ base_url: "http://localhost:8080"
2222
# scanning. Set to "0" to disable. Default: "5m". Does not limit proxy mirror.
2323
# artifact_timeout: "5m"
2424

25-
# Register DELETE /api/cache and show Delete buttons on the UI's version page,
26-
# for removing cached artifacts so they are fetched again. Has no
27-
# authentication: restrict /api/cache and /ui at the ingress first.
25+
# Register DELETE /api/cache and, with upstream.git set, DELETE /api/git, and
26+
# show Delete buttons on the UI's version and git pages, for removing cached
27+
# artifacts and repositories so they are fetched again. Has no authentication:
28+
# restrict /api/cache, /api/git and /ui at the ingress first.
2829
# cache_delete: false
2930

3031
# Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused

‎docs/configuration.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1888,7 +1888,7 @@ cache_delete: true
18881888

18891889
Or via environment variable: `PROXY_CACHE_DELETE=true`.
18901890

1891-
The endpoint has no authentication. Before enabling it, restrict `/api/cache` and `/ui` to trusted users at your ingress or reverse proxy. The UI sends its deletes to `/ui/api/cache`, so whatever protects `/ui` protects those too.
1891+
The endpoints have no authentication. Before enabling them, restrict `/api/cache`, `/api/git` and `/ui` to trusted users at your ingress or reverse proxy. The UI sends its deletes to `/ui/api/cache` and `/ui/api/git`, so whatever protects `/ui` protects those too.
18921892

18931893
Delete every cached file of a version, or only the one named by `filename`:
18941894

@@ -1901,6 +1901,14 @@ The name is the package name the UI shows, for example `group:artifact` for Mave
19011901

19021902
Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages).
19031903

1904+
With `upstream.git` set, `DELETE /api/git/{upstream}/{repo}` removes one repository from the [git cache](#git-repositories), and the `/ui/git` page gets a Delete button per repository:
1905+
1906+
```bash
1907+
curl -X DELETE http://localhost:8080/api/git/github/acme/widgets.git
1908+
```
1909+
1910+
`{upstream}` is the name under `upstream.git`, and `{repo}` the repository path with or without `.git`. The response names the repository and the bytes freed (`freed_bytes`). The next clone fetches it from the upstream again. A repository that a clone, a fetch or a client is using at that moment answers 409; try again once it is done. Git LFS objects are deleted through `/api/cache` like other artifacts.
1911+
19041912
## Mirror Command
19051913

19061914
The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`.

‎docs/swagger/docs.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,54 @@ const docTemplate = `{
182182
}
183183
}
184184
},
185+
"/api/git/{upstream}/{repo}": {
186+
"delete": {
187+
"description": "Removes one repository from the git cache, so the next clone fetches it from the upstream again. A repository that a clone, a fetch or a client is using answers 409. Needs cache_delete.",
188+
"produces": [
189+
"application/json"
190+
],
191+
"tags": [
192+
"cache"
193+
],
194+
"summary": "Delete a cached Git repository",
195+
"parameters": [
196+
{
197+
"type": "string",
198+
"description": "Name of the upstream.git entry",
199+
"name": "upstream",
200+
"in": "path",
201+
"required": true
202+
},
203+
{
204+
"type": "string",
205+
"description": "Repository path under the upstream, with or without .git",
206+
"name": "repo",
207+
"in": "path",
208+
"required": true
209+
}
210+
],
211+
"responses": {
212+
"200": {
213+
"description": "OK",
214+
"schema": {
215+
"$ref": "#/definitions/server.GitDeleteResponse"
216+
}
217+
},
218+
"404": {
219+
"description": "Not Found",
220+
"schema": {
221+
"$ref": "#/definitions/server.ErrorResponse"
222+
}
223+
},
224+
"409": {
225+
"description": "Conflict",
226+
"schema": {
227+
"$ref": "#/definitions/server.ErrorResponse"
228+
}
229+
}
230+
}
231+
}
232+
},
185233
"/api/hosted/{ecosystem}/{name}": {
186234
"get": {
187235
"produces": [
@@ -1074,6 +1122,17 @@ const docTemplate = `{
10741122
}
10751123
}
10761124
},
1125+
"server.GitDeleteResponse": {
1126+
"type": "object",
1127+
"properties": {
1128+
"freed_bytes": {
1129+
"type": "integer"
1130+
},
1131+
"repository": {
1132+
"type": "string"
1133+
}
1134+
}
1135+
},
10771136
"server.HealthCheck": {
10781137
"type": "object",
10791138
"properties": {

‎docs/swagger/swagger.json‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,54 @@
175175
}
176176
}
177177
},
178+
"/api/git/{upstream}/{repo}": {
179+
"delete": {
180+
"description": "Removes one repository from the git cache, so the next clone fetches it from the upstream again. A repository that a clone, a fetch or a client is using answers 409. Needs cache_delete.",
181+
"produces": [
182+
"application/json"
183+
],
184+
"tags": [
185+
"cache"
186+
],
187+
"summary": "Delete a cached Git repository",
188+
"parameters": [
189+
{
190+
"type": "string",
191+
"description": "Name of the upstream.git entry",
192+
"name": "upstream",
193+
"in": "path",
194+
"required": true
195+
},
196+
{
197+
"type": "string",
198+
"description": "Repository path under the upstream, with or without .git",
199+
"name": "repo",
200+
"in": "path",
201+
"required": true
202+
}
203+
],
204+
"responses": {
205+
"200": {
206+
"description": "OK",
207+
"schema": {
208+
"$ref": "#/definitions/server.GitDeleteResponse"
209+
}
210+
},
211+
"404": {
212+
"description": "Not Found",
213+
"schema": {
214+
"$ref": "#/definitions/server.ErrorResponse"
215+
}
216+
},
217+
"409": {
218+
"description": "Conflict",
219+
"schema": {
220+
"$ref": "#/definitions/server.ErrorResponse"
221+
}
222+
}
223+
}
224+
}
225+
},
178226
"/api/hosted/{ecosystem}/{name}": {
179227
"get": {
180228
"produces": [
@@ -1067,6 +1115,17 @@
10671115
}
10681116
}
10691117
},
1118+
"server.GitDeleteResponse": {
1119+
"type": "object",
1120+
"properties": {
1121+
"freed_bytes": {
1122+
"type": "integer"
1123+
},
1124+
"repository": {
1125+
"type": "string"
1126+
}
1127+
}
1128+
},
10701129
"server.HealthCheck": {
10711130
"type": "object",
10721131
"properties": {

‎internal/config/config.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,9 +212,10 @@ type Config struct {
212212
// Mirror configures scheduled mirroring.
213213
Mirror MirrorConfig `json:"mirror" yaml:"mirror"`
214214

215-
// CacheDelete enables DELETE /api/cache for removing cached artifacts, and
216-
// the delete buttons on the UI's version page. It has no authentication of
217-
// its own, so restrict /api/cache and /ui at the ingress when enabling it.
215+
// CacheDelete enables DELETE /api/cache and DELETE /api/git for removing
216+
// cached artifacts and git repositories, and the delete buttons on the
217+
// UI's version and git pages. It has no authentication of its own, so
218+
// restrict /api/cache, /api/git and /ui at the ingress when enabling it.
218219
CacheDelete bool `json:"cache_delete" yaml:"cache_delete"`
219220

220221
// Hosted configures pushing packages to the proxy (/api/hosted).

‎internal/gitcache/remove.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package gitcache
2+
3+
import (
4+
"errors"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
)
9+
10+
var (
11+
// ErrNotCached is returned by Remove for a repository the cache does
12+
// not hold.
13+
ErrNotCached = errors.New("repository is not cached")
14+
// ErrInUse is returned by Remove for a repository a clone, a fetch or
15+
// a client is using.
16+
ErrInUse = errors.New("repository is in use")
17+
)
18+
19+
// Remove deletes the cached repository under key, as eviction does, and
20+
// returns its size. The next request for it clones it again. A repository
21+
// in use is not removed and returns ErrInUse, so nothing reads it while it
22+
// is deleted.
23+
func (c *Cache) Remove(key string) (int64, error) {
24+
c.mu.Lock()
25+
rp := c.repos[c.repoKey(key)]
26+
if rp == nil {
27+
c.mu.Unlock()
28+
return 0, ErrNotCached
29+
}
30+
// Before the directory check: a first clone is built elsewhere and
31+
// moved into place once done.
32+
if rp.users > 0 {
33+
c.mu.Unlock()
34+
return 0, ErrInUse
35+
}
36+
if !rp.exists() {
37+
c.mu.Unlock()
38+
return 0, ErrNotCached
39+
}
40+
// Moved aside under the lock, so a request that comes after it clones
41+
// the repository again instead of reading it while it is deleted.
42+
trash, err := os.MkdirTemp(c.tmpDir(), "removed-")
43+
if err == nil {
44+
err = os.Rename(rp.dir, filepath.Join(trash, "repo"))
45+
}
46+
if err != nil {
47+
c.mu.Unlock()
48+
if trash != "" {
49+
_ = os.Remove(trash)
50+
}
51+
return 0, err
52+
}
53+
delete(c.repos, rp.key)
54+
delete(c.tooLarge, rp.key)
55+
// A want the old copy could not fit may fit in the next clone.
56+
for k := range c.failedWants {
57+
if strings.HasPrefix(k, rp.key+" ") {
58+
delete(c.failedWants, k)
59+
}
60+
}
61+
size := rp.size.Load()
62+
c.mu.Unlock()
63+
64+
c.logger.Info("git cache: removed repository", "repository", rp.key, "size", size)
65+
c.removeEvicted([]string{trash})
66+
return size, nil
67+
}

‎internal/gitcache/remove_test.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package gitcache
2+
3+
import (
4+
"errors"
5+
"path/filepath"
6+
"testing"
7+
)
8+
9+
func TestRemoveDeletesARepository(t *testing.T) {
10+
upstream := newFakeUpstream(t)
11+
upstream.newRepo("repo")
12+
tc := newTestCache(t, upstream.server.URL, nil)
13+
tc.clone(t, "repo")
14+
waitIdle(t, tc.Cache)
15+
dir := filepath.Join(tc.dir, "up", "repo.git")
16+
want := tc.repo("up/repo.git").size.Load()
17+
18+
freed, err := tc.Remove("up/repo.git")
19+
if err != nil {
20+
t.Fatalf("Remove: %v", err)
21+
}
22+
if freed != want || freed == 0 {
23+
t.Errorf("freed %d bytes, want %d", freed, want)
24+
}
25+
if fileExists(dir) || tc.repo("up/repo.git") != nil {
26+
t.Error("the repository is still cached")
27+
}
28+
if len(tc.Repositories()) != 0 {
29+
t.Errorf("Repositories() = %+v, want none", tc.Repositories())
30+
}
31+
if _, err := tc.Remove("up/repo.git"); !errors.Is(err, ErrNotCached) {
32+
t.Errorf("second Remove: %v, want ErrNotCached", err)
33+
}
34+
35+
// The next clone fetches it again.
36+
requests := upstream.gitRequests.Load()
37+
tc.clone(t, "repo")
38+
if upstream.gitRequests.Load() == requests || !fileExists(dir) {
39+
t.Error("the clone after Remove did not fetch the repository again")
40+
}
41+
}
42+
43+
// A repository that a clone, a fetch or a client is using is not removed,
44+
// so nothing reads it while it is deleted.
45+
func TestRemoveRefusesARepositoryInUse(t *testing.T) {
46+
upstream := newFakeUpstream(t)
47+
upstream.newRepo("repo")
48+
tc := newTestCache(t, upstream.server.URL, nil)
49+
tc.clone(t, "repo")
50+
waitIdle(t, tc.Cache)
51+
52+
rp := tc.acquire("up/repo.git")
53+
if _, err := tc.Remove("up/repo.git"); !errors.Is(err, ErrInUse) {
54+
t.Errorf("Remove while in use: %v, want ErrInUse", err)
55+
}
56+
tc.release(rp)
57+
if !fileExists(filepath.Join(tc.dir, "up", "repo.git")) {
58+
t.Fatal("the repository was removed while in use")
59+
}
60+
if _, err := tc.Remove("up/repo.git"); err != nil {
61+
t.Errorf("Remove once idle: %v", err)
62+
}
63+
}
64+
65+
func TestRemoveUnknownRepository(t *testing.T) {
66+
upstream := newFakeUpstream(t)
67+
tc := newTestCache(t, upstream.server.URL, nil)
68+
for _, key := range []string{"up/missing.git", "up/../up.git", ""} {
69+
if _, err := tc.Remove(key); !errors.Is(err, ErrNotCached) {
70+
t.Errorf("Remove(%q): %v, want ErrNotCached", key, err)
71+
}
72+
}
73+
}
74+
75+
// A repository whose first clone is still running is in use, not missing.
76+
func TestRemoveRefusesARepositoryBeingCloned(t *testing.T) {
77+
upstream := newFakeUpstream(t)
78+
upstream.newRepo("repo")
79+
tc := newTestCache(t, upstream.server.URL, nil)
80+
arrived, release := upstream.holdFetches()
81+
cloned := make(chan error, 1)
82+
go func() {
83+
_, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), filepath.Join(t.TempDir(), "clone"))
84+
cloned <- err
85+
}()
86+
<-arrived
87+
if _, err := tc.Remove("up/repo.git"); !errors.Is(err, ErrInUse) {
88+
t.Errorf("Remove during the first clone: %v, want ErrInUse", err)
89+
}
90+
release()
91+
if err := <-cloned; err != nil {
92+
t.Fatalf("clone: %v", err)
93+
}
94+
}
95+
96+
// Wants a removed repository could not get may fit once it is cloned again,
97+
// so removing it clears them, and only them.
98+
func TestRemoveForgetsTheRepositorysFailedWants(t *testing.T) {
99+
upstream := newFakeUpstream(t)
100+
upstream.newRepo("repo")
101+
tc := newTestCache(t, upstream.server.URL, nil)
102+
tc.clone(t, "repo")
103+
waitIdle(t, tc.Cache)
104+
oid := "0123456789abcdef0123456789abcdef01234567"
105+
tc.recordFailedWants("up/repo.git", []string{oid}, ErrTooLarge)
106+
tc.recordFailedWants("up/other.git", []string{oid}, ErrTooLarge)
107+
108+
if _, err := tc.Remove("up/repo.git"); err != nil {
109+
t.Fatalf("Remove: %v", err)
110+
}
111+
if got := tc.withoutFailedWants("up/repo.git", []string{oid}); len(got) != 1 {
112+
t.Error("the removed repository's failed want is still held back")
113+
}
114+
if got := tc.withoutFailedWants("up/other.git", []string{oid}); len(got) != 0 {
115+
t.Error("another repository's failed want was cleared")
116+
}
117+
}

0 commit comments

Comments
 (0)