fix(vcs): discover private repositories of personal accounts (#42)

git-sync listed a GitHub personal account through GET /users/{owner}/repos,
which returns public repositories only whatever token is presented, so private
repositories were silently missing from every run with no error to point at.
When the token belongs to the configured owner, list through
GET /user/repos?visibility=all&affiliation=owner instead, which reports them.
Mirroring another account still sees its public repositories only, since no
credential can see further.

A GitLab source in a user namespace failed outright at the discover stage,
because only GET /groups/{owner}/projects was tried and it 404s for a personal
namespace. A 404 on the first page now falls back to
GET /users/{owner}/projects.

A mirror whose discovery succeeds but reports nothing now warns and names the
likely cause, since an empty run was otherwise indistinguishable from a mirror
that is up to date.

Co-authored-by: Anmol Nagpal <ianmolnagpal@gmail.com>

Author: clouddrove-ci

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- `git-sync` discovered only the public repositories of a GitHub personal account. The listing fell back to `GET /users/{owner}/repos` when the owner was not an organisation, and that endpoint returns public repositories only, whatever token is presented, so private repositories were silently missing from every run with no error to point at. When the token belongs to the configured owner, SyncerD now lists through `GET /user/repos?visibility=all&affiliation=owner`, which reports private repositories. Mirroring another account still sees its public repositories only, since no credential can see further
+- `git-sync` failed outright at the `discover` stage when a GitLab source was a user namespace rather than a group: only `GET /groups/{owner}/projects` was tried, and it 404s for a personal namespace. A 404 on the first page now falls back to `GET /users/{owner}/projects`, which lists private projects the token can see
+
+### Changed
+- A `git-sync` mirror whose source discovery succeeds but reports no repositories at all now logs a warning naming the likely cause (a token that cannot read the source's private repositories), because an empty run was otherwise indistinguishable from a mirror that is up to date
+
 ## [0.1.2] - 2026-08-14
 
 ### Added

docs/git-sync-runbook.md

@@ -210,7 +210,7 @@ query and the one-file-per-command note.
 
 | Provider | Token | Scopes |
 |---|---|---|
-| GitHub | fine-grained PAT | Metadata read, Contents read, Administration write (only if `create_missing`) |
+| GitHub | fine-grained PAT | Metadata read, Contents read, Administration write (only if `create_missing`); a fine grained token must also grant access to each private repository you expect to be mirrored, or discovery reports fewer repositories than the source holds |
 | GitLab | group access token | `api` to create projects, plus `write_repository` |
 
 Environment variable names derive from the **provider name** in your config,
@@ -576,6 +576,17 @@ config that looks valid can still fail at the credential or network step:
   Bitbucket also has no archived concept, so `skip_archived` has no effect
   for a Bitbucket source.
 
+**A personal account source needs a token belonging to that account.**
+GitHub's `GET /users/{owner}/repos` returns public repositories only, whatever
+token is presented, so SyncerD lists a personal account through
+`GET /user/repos` instead, which reports private repositories too. That
+endpoint only ever describes the account the token belongs to. Mirroring
+someone else's personal account therefore discovers its public repositories
+and nothing more, because no credential can see further. Organisation owners
+are unaffected: `GET /orgs/{org}/repos` reports private repositories to any
+token with access. The same shape applies to GitLab, where a user namespace
+falls back to `GET /users/{owner}/projects`.
+
 **Released binaries need git on PATH.** goreleaser ships a bare binary with
 no bundled git; install git 2.30 or newer on any host that runs a release
 binary. The Docker image installs it for you.

internal/gitsync/sync.go

@@ -214,6 +214,16 @@ func (e *Engine) Run(ctx context.Context, mirrors []Mirror) (*GitReport, error)
 		logging.Info(fmt.Sprintf("mirror %s: %d repositories discovered, %d selected", m.Name, len(repos), len(selected)),
 			"mirror", m.Name, "discovered", len(repos), "selected", len(selected))
 
+		// Discovery that succeeds but reports nothing is the shape a
+		// under scoped token takes: the API answers, and answers with the
+		// public half of an account, or with nothing at all. Say so, since
+		// an empty run is otherwise indistinguishable from a mirror that
+		// is genuinely up to date.
+		if len(repos) == 0 {
+			logging.Warn(fmt.Sprintf("mirror %s: source provider reported no repositories at all; if the source holds private repositories, check that the token can read them (a GitHub token needs the repo scope, or repository access on a fine grained token; a GitLab token needs read_api)", m.Name),
+				"mirror", m.Name)
+		}
+
 		if err := e.checkDestinationCollisions(m, selected); err != nil {
 			e.addFailure(GitFailure{Mirror: m.Name, Stage: "discover", Error: e.redact(err.Error())})
 			if e.opts.FailFast {

internal/vcs/github/github.go

@@ -42,6 +42,9 @@ type Provider struct {
 
 	ownerMu    sync.Mutex
 	ownerIsOrg *bool
+
+	loginMu sync.Mutex
+	login   *string
 }
 
 // New validates cfg and returns a provider.
@@ -154,13 +157,71 @@ func (p *Provider) ListRepos(ctx context.Context) ([]vcs.Repo, error) {
 	// A 404 partway through is a real failure, not a reason to restart
 	// against a different endpoint and lose what we already collected.
 	if isNotFound(err) && len(repos) == 0 {
-		userURL := fmt.Sprintf("%s/users/%s/repos?per_page=100&type=all", p.apiURL, p.owner)
-		return p.listFrom(ctx, userURL)
+		return p.listUserRepos(ctx)
 	}
 
 	return nil, err
 }
 
+// listUserRepos lists a personal account's repositories.
+//
+// GET /users/{username}/repos returns public repositories only, whatever
+// the token: GitHub documents it as "Lists public repositories for the
+// specified user", and it does not widen for an authenticated caller. A
+// mirror of a personal account configured that way silently discovers only
+// the public half, with no error to point at.
+//
+// GET /user/repos is the endpoint that reports private repositories, but it
+// only ever describes the account the token belongs to. So when the token
+// belongs to the configured owner, that endpoint is used; otherwise the
+// public listing is all any credential can see anyway.
+func (p *Provider) listUserRepos(ctx context.Context) ([]vcs.Repo, error) {
+	login, err := p.authenticatedLogin(ctx)
+	if err != nil {
+		// The token could not be resolved to an account. Fall back to the
+		// public listing rather than failing discovery outright, so a
+		// mirror of a public personal account keeps working.
+		login = ""
+	}
+
+	if login != "" && strings.EqualFold(login, p.owner) {
+		selfURL := fmt.Sprintf("%s/user/repos?per_page=100&affiliation=owner&visibility=all", p.apiURL)
+		return p.listFrom(ctx, selfURL)
+	}
+
+	userURL := fmt.Sprintf("%s/users/%s/repos?per_page=100&type=all", p.apiURL, p.owner)
+	return p.listFrom(ctx, userURL)
+}
+
+// authenticatedLogin returns the account login the token belongs to. The
+// answer is cached because it cannot change during a run, and the lock is
+// held across the lookup so concurrent workers resolve it once. A token
+// that GET /user does not accept, such as a GitHub App installation token,
+// yields an error rather than a login.
+func (p *Provider) authenticatedLogin(ctx context.Context) (string, error) {
+	p.loginMu.Lock()
+	defer p.loginMu.Unlock()
+
+	if p.login != nil {
+		return *p.login, nil
+	}
+
+	body, _, err := p.do(ctx, http.MethodGet, p.apiURL+"/user", nil)
+	if err != nil {
+		// Not cached, so a later call can retry.
+		return "", err
+	}
+
+	var who struct {
+		Login string `json:"login"`
+	}
+	if err := json.Unmarshal(body, &who); err != nil {
+		return "", fmt.Errorf("github: decode authenticated user: %w", err)
+	}
+	p.login = &who.Login
+	return who.Login, nil
+}
+
 // maxPages bounds a paginated listing. GitHub caps at 100 per page, so this
 // allows 100000 repositories, far beyond any real org, while stopping a
 // server that returns a cyclic next link.

internal/vcs/github/github_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"sync"
 	"testing"
@@ -121,6 +122,64 @@ func TestListReposFallsBackToUserEndpoint(t *testing.T) {
 	}
 }
 
+func TestListReposUsesAuthenticatedEndpointForTheTokenOwner(t *testing.T) {
+	var query url.Values
+	mux := http.NewServeMux()
+	mux.HandleFunc("/orgs/acme/repos", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"Not Found"}`, http.StatusNotFound)
+	})
+	mux.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
+		writeJSON(w, map[string]any{"login": "Acme"})
+	})
+	mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) {
+		query = r.URL.Query()
+		writeJSON(w, []map[string]any{{"name": "secret", "full_name": "acme/secret", "default_branch": "main"}})
+	})
+	mux.HandleFunc("/users/acme/repos", func(w http.ResponseWriter, r *http.Request) {
+		t.Error("the public user listing hides private repositories and must not be used for the token owner")
+	})
+
+	p, _ := newProvider(t, mux)
+	repos, err := p.ListRepos(context.Background())
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(repos) != 1 || repos[0].Name != "secret" {
+		t.Fatalf("unexpected repos: %+v", repos)
+	}
+	if got := query.Get("visibility"); got != "all" {
+		t.Errorf("visibility = %q, want all", got)
+	}
+	if got := query.Get("affiliation"); got != "owner" {
+		t.Errorf("affiliation = %q, want owner", got)
+	}
+}
+
+func TestListReposUsesPublicEndpointForAnotherAccount(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/orgs/acme/repos", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"Not Found"}`, http.StatusNotFound)
+	})
+	mux.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
+		writeJSON(w, map[string]any{"login": "someone-else"})
+	})
+	mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) {
+		t.Error("the authenticated listing describes the token owner, not the configured owner")
+	})
+	mux.HandleFunc("/users/acme/repos", func(w http.ResponseWriter, r *http.Request) {
+		writeJSON(w, []map[string]any{{"name": "personal", "full_name": "acme/personal", "default_branch": "main"}})
+	})
+
+	p, _ := newProvider(t, mux)
+	repos, err := p.ListRepos(context.Background())
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(repos) != 1 || repos[0].Name != "personal" {
+		t.Fatalf("unexpected repos: %+v", repos)
+	}
+}
+
 func TestListReposSurfacesServerError(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/orgs/acme/repos", func(w http.ResponseWriter, r *http.Request) {

internal/vcs/gitlab/gitlab.go

@@ -128,6 +128,38 @@ const maxPages = 1000
 // returns a cyclic or repeated page number cannot loop it forever; whatever
 // was collected before a bound trip is still returned alongside the error.
 func (p *Provider) ListRepos(ctx context.Context) ([]vcs.Repo, error) {
+	groupPage := func(page string) string {
+		return fmt.Sprintf("%s/api/v4/groups/%s/projects?include_subgroups=true&per_page=100&archived=&page=%s",
+			p.base, url.PathEscape(p.owner), url.QueryEscape(page))
+	}
+
+	repos, err := p.listPages(ctx, groupPage)
+	if err == nil {
+		return repos, nil
+	}
+
+	// A 404 on the very first page means the owner is a user namespace
+	// rather than a group. GET /users/{owner}/projects lists that
+	// namespace, including its private projects when the token can see
+	// them, so a personal namespace is mirrorable instead of failing
+	// discovery outright. A 404 partway through is a real failure, not a
+	// reason to restart against a different endpoint.
+	if isNotFound(err) && len(repos) == 0 {
+		userPage := func(page string) string {
+			return fmt.Sprintf("%s/api/v4/users/%s/projects?per_page=100&archived=&page=%s",
+				p.base, url.PathEscape(p.owner), url.QueryEscape(page))
+		}
+		return p.listPages(ctx, userPage)
+	}
+
+	return nil, err
+}
+
+// listPages walks a paginated project listing built by pageURL, following
+// the X-Next-Page header. Whatever was collected before an error is still
+// returned alongside it, so the caller can tell a first page failure from a
+// failure partway through.
+func (p *Provider) listPages(ctx context.Context, pageURL func(page string) string) ([]vcs.Repo, error) {
 	var out []vcs.Repo
 	page := "1"
 	seen := make(map[string]bool)
@@ -141,10 +173,7 @@ func (p *Provider) ListRepos(ctx context.Context) ([]vcs.Repo, error) {
 		}
 		seen[page] = true
 
-		endpoint := fmt.Sprintf("%s/api/v4/groups/%s/projects?include_subgroups=true&per_page=100&archived=&page=%s",
-			p.base, url.PathEscape(p.owner), url.QueryEscape(page))
-
-		body, header, err := p.do(ctx, http.MethodGet, endpoint, nil)
+		body, header, err := p.do(ctx, http.MethodGet, pageURL(page), nil)
 		if err != nil {
 			return out, err
 		}

internal/vcs/gitlab/gitlab_test.go

@@ -97,6 +97,42 @@ func TestListReposFallsBackToComputedCloneURLWhenAPIOmitsIt(t *testing.T) {
 	}
 }
 
+func TestListReposFallsBackToUserNamespace(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/v4/groups/acme/projects", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"404 Group Not Found"}`, http.StatusNotFound)
+	})
+	mux.HandleFunc("/api/v4/users/acme/projects", func(w http.ResponseWriter, r *http.Request) {
+		writeJSON(w, []map[string]any{
+			{"path": "personal", "path_with_namespace": "acme/personal", "default_branch": "main"},
+		})
+	})
+
+	p := newProvider(t, mux)
+	repos, err := p.ListRepos(context.Background())
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(repos) != 1 || repos[0].Name != "personal" {
+		t.Fatalf("unexpected repos: %+v", repos)
+	}
+}
+
+func TestListReposDoesNotFallBackOnNonNotFoundError(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/v4/groups/acme/projects", func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"401 Unauthorized"}`, http.StatusUnauthorized)
+	})
+	mux.HandleFunc("/api/v4/users/acme/projects", func(w http.ResponseWriter, r *http.Request) {
+		t.Error("a 401 on the group endpoint must not silently fall back to the user namespace")
+	})
+
+	p := newProvider(t, mux)
+	if _, err := p.ListRepos(context.Background()); err == nil {
+		t.Fatal("expected the 401 to be surfaced")
+	}
+}
+
 func TestEnsureRepoLooksUpNamespaceThenCreates(t *testing.T) {
 	var created map[string]any
 	mux := http.NewServeMux()