Merge pull request #28 from gxc/ssh-git-servers

feat: allow SSH clones from custom git servers via GIT_CLONE_EXTRA_HOSTS

Author: Lekssays

.env.example

@@ -82,6 +82,38 @@ CPG_QUEUE_BACKEND=durable
 # Empty = no allowlist.
 # ALLOWED_SOURCE_ROOTS=/abs/path/to/sources:/abs/path/to/other-sources
 
+# Custom git clone servers (optional) — allow your own git host beyond
+# github.com/gitlab.com, e.g. a self-hosted Forgejo on the LAN.
+# GIT_CLONE_EXTRA_HOSTS: ','-separated `host[:port]` entries. A bare host means
+# port 22 only; `host:port` pins that port; `host:*` allows any port on it.
+# Listed hosts are cloned via ssh:// only (github.com/gitlab.com stay
+# https-only). A malformed value fails the server boot.
+# ssh:// auth depends on how the MCP runs (the clone runs where the MCP runs):
+# - MCP on the host (`python main.py`): set GIT_CLONE_SSH_KEY_PATH to the key
+#   FILE on this host, or a full GIT_CLONE_SSH_COMMAND override.
+# - Full docker stack (`./scripts/deploy.sh`): set GIT_CLONE_SSH_KEYS_HOST_DIR
+#   to a HOST directory containing the private key as id_ed25519; compose mounts
+#   it read-only at /keys in the codebadger-mcp container and the server then
+#   sees /keys/id_ed25519. GIT_CLONE_SSH_KEY_PATH has no effect in the container
+#   — do not set it here. With GIT_CLONE_SSH_COMMAND, any path it references
+#   must exist inside the container (e.g. /keys/...).
+# Example Forgejo reachable at ssh://git@192.168.152.14:3000/...:
+#   GIT_CLONE_EXTRA_HOSTS=192.168.152.14:3000
+#   # host-run MCP:
+#   GIT_CLONE_SSH_KEY_PATH=/path/to/id_ed25519
+#   # dockerized stack:
+#   GIT_CLONE_SSH_KEYS_HOST_DIR=/path/to/keydir   # containing id_ed25519
+# Host keys: without GIT_CLONE_SSH_KNOWN_HOSTS the clone uses
+# StrictHostKeyChecking=accept-new (trust-on-first-use — and in the dockerized
+# stack that record dies with the container, so it is TOFU on every recreate).
+# Point it at a known_hosts file to pin the server key instead; in the docker
+# stack that path must be IN-CONTAINER, e.g. /keys/known_hosts next to the key.
+# GIT_CLONE_EXTRA_HOSTS=
+# GIT_CLONE_SSH_KEYS_HOST_DIR=
+# GIT_CLONE_SSH_KEY_PATH=
+# GIT_CLONE_SSH_KNOWN_HOSTS=
+# GIT_CLONE_SSH_COMMAND=
+
 DOCKER_HOST=unix:///var/run/docker.sock
 
 # GitHub (optional) — token for cloning private repos.

.gitignore

@@ -145,3 +145,9 @@ codebadger.db
 docker-compose.override.yml
 scripts/backfill_overlays.sh
 .claude/settings.json
+
+# Operator ssh keys for GIT_CLONE_EXTRA_HOSTS clones (compose default host dir).
+# Never commit private keys. The dir itself is tracked (via .gitkeep) so compose
+# doesn't create it root-owned when GIT_CLONE_SSH_KEYS_HOST_DIR is unset.
+.ssh-keys/*
+!.ssh-keys/.gitkeep

.ssh-keys/.gitkeep

@@ -0,0 +1,3 @@
+# Default mount source for GIT_CLONE_SSH_KEYS_HOST_DIR (docker-compose).
+# Place an operator deploy key here as id_ed25519 (and optionally known_hosts).
+# Everything in this directory except this file is gitignored.

Dockerfile.mcp

@@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         curl \
         ca-certificates \
+        openssh-client \
     && curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_CLI_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \
         | tar -xz -C /usr/local/bin --strip-components=1 docker/docker \
     && docker --version \

docker-compose.yml

@@ -54,6 +54,13 @@ services:
       # DOCKER_HOST) for a rootless / non-default socket; container side stays fixed.
       - ${DOCKER_SOCK:-/var/run/docker.sock}:/var/run/docker.sock
       - ./logs:/app/logs
+      # Operator ssh key dir for GIT_CLONE_EXTRA_HOSTS clones, mounted READ-ONLY
+      # at the fixed in-container path /keys. Point GIT_CLONE_SSH_KEYS_HOST_DIR
+      # (.env) at a HOST dir containing the private key as id_ed25519; the key
+      # then resolves to /keys/id_ed25519 inside this container (see the env
+      # below). Unset => an empty dir is mounted and no key path is set — no
+      # behavior change.
+      - ${GIT_CLONE_SSH_KEYS_HOST_DIR:-./.ssh-keys}:/keys:ro
     environment:
       # 0.0.0.0 = reachable on all interfaces. Set MCP_HOST=127.0.0.1 to bind
       # loopback only (e.g. when a reverse proxy / socat already fronts it).
@@ -96,6 +103,21 @@ services:
       # this container — local sources live under /app/playground here.
       CHAT_DEPLOY: ${CHAT_DEPLOY:-false}
       ALLOWED_SOURCE_ROOTS: ${ALLOWED_SOURCE_ROOTS:-}
+      # Custom git clone servers (optional) — allowlist your own git host
+      # (e.g. a LAN Forgejo) beyond github.com/gitlab.com. ssh:// clones auth
+      # via the mounted key (below) or a full GIT_CLONE_SSH_COMMAND.
+      GIT_CLONE_EXTRA_HOSTS: ${GIT_CLONE_EXTRA_HOSTS:-}
+      # In the dockerized stack the key's in-container path is DERIVED from
+      # GIT_CLONE_SSH_KEYS_HOST_DIR (the host dir mounted at /keys above).
+      # GIT_CLONE_SSH_KEY_PATH is a host-run-MCP setting and is deliberately
+      # NOT passed through here — a host path would never resolve in-container.
+      GIT_CLONE_SSH_KEY_PATH: ${GIT_CLONE_SSH_KEYS_HOST_DIR:+/keys/id_ed25519}
+      # Optional host-key pinning. Like GIT_CLONE_SSH_COMMAND this is an
+      # IN-CONTAINER path: drop a known_hosts next to the key and set
+      # GIT_CLONE_SSH_KNOWN_HOSTS=/keys/known_hosts. Unset => accept-new, whose
+      # record lives in the container and is lost on every recreate.
+      GIT_CLONE_SSH_KNOWN_HOSTS: ${GIT_CLONE_SSH_KNOWN_HOSTS:-}
+      GIT_CLONE_SSH_COMMAND: ${GIT_CLONE_SSH_COMMAND:-}
     depends_on:
       codebadger-postgres:
         condition: service_healthy

docs/deployment.md

@@ -139,11 +139,13 @@ CHAT_DEPLOY=true     # in .env (passed through to the container by compose)
 With `CHAT_DEPLOY=true` the MCP **refuses `source_type="local"`** and returns a
 message steering the caller to the safe inputs. What remains:
 
-- **Git repos** — only `https://github.com/…` and `https://gitlab.com/…` are
-  accepted. The URL is checked twice (a literal `https://<host>/` prefix *and* a
-  parsed-hostname allowlist) and rejects other hosts, non-`https` schemes
-  (`git://`, `ssh://`, `file://`), embedded credentials, ports, and look-alike
-  domains — so a repo URL can't be turned into an SSRF probe.
+- **Git repos** — `https://github.com/…` / `https://gitlab.com/…` are accepted,
+  plus `ssh://…` on hosts the operator listed in `GIT_CLONE_EXTRA_HOSTS`
+  (see [Custom git servers](#custom-git-servers-git_clone_) below). The URL is
+  checked twice (a literal `https://<host>/` prefix *and* a parsed-hostname
+  allowlist) and rejects other hosts, non-`https` schemes (`git://`, `file://`),
+  embedded credentials, ports, and look-alike domains — so a repo URL can't be
+  turned into an SSRF probe.
 - **Pasted snippets** — `source_type="snippet"` with the code in a
   `<code language="…">` tag; the language is validated/inferred and a mislabeled
   or ambiguous snippet is refused. Nothing touches the host filesystem.
@@ -205,6 +207,66 @@ defaults — note a few differ in the shipped `docker-compose.yml` (called out b
 | `ALLOWED_SOURCE_ROOTS` | `` (empty) | `:`-separated allowlist of dirs local sources must canonically resolve within (as the MCP sees them, e.g. `/app/playground`). Empty = no allowlist. |
 | `GITHUB_TOKEN` | `` (empty) | PAT for cloning private repos (never embed it in the URL). |
 
+### Custom git servers (`GIT_CLONE_*`)
+
+By default `generate_cpg` only clones from `https://github.com/…` /
+`https://gitlab.com/…`. To also analyze code on your own git server (e.g. a
+self-hosted **Forgejo**/**Gitea** on the LAN), allowlist it with
+`GIT_CLONE_EXTRA_HOSTS` — no other change is needed; callers then pass the
+repo URL (`ssh://git@192.168.152.14:3000/<owner>/<repo>.git`) as
+`source_path` with `source_type='github'`. Custom hosts are **ssh-only** —
+http(s) clone URLs are rejected for them.
+
+```bash
+# ','-separated host[:port] entries — the same in both modes below.
+# A bare host means port 22; `host:port` pins that port; `host:*` allows any.
+GIT_CLONE_EXTRA_HOSTS=192.168.152.14:3000
+
+# MCP run on the host (`python main.py`): key FILE on this host
+GIT_CLONE_SSH_KEY_PATH=/abs/path/to/id_ed25519
+
+# Full docker stack (`./scripts/deploy.sh`): HOST DIRECTORY containing the key
+# as id_ed25519; compose mounts it read-only at /keys in the codebadger-mcp
+# container, so the server sees /keys/id_ed25519
+GIT_CLONE_SSH_KEYS_HOST_DIR=/abs/path/to/keydir
+```
+
+| Variable | Default | Description |
+|---|---|---|
+| `GIT_CLONE_EXTRA_HOSTS` | `` (empty) | ','-separated `host[:port]` entries accepted in addition to github.com/gitlab.com (IPv6 goes in brackets, e.g. `[::1]:2222`). A bare host means **port 22 only**; `host:port` pins that port; `host:*` allows any port on it. Allowlisted hosts are cloned over `ssh://` only; the built-in hosts keep their strict https-only, default-port-only posture unless an operator explicitly lists one here (e.g. `github.com:22` would enable ssh for github.com). Parsed once at startup — a malformed value fails the boot rather than surfacing on the first ssh clone. |
+| `GIT_CLONE_SSH_KEYS_HOST_DIR` | `` (empty) | **Dockerized stack only.** Host directory containing the private key (named `id_ed25519`); docker compose mounts it read-only at `/keys` in the `codebadger-mcp` container and the server resolves the key to the fixed in-container path `/keys/id_ed25519`. Unset, an empty dir is mounted and no key path is configured. |
+| `GIT_CLONE_SSH_KEY_PATH` | `` (empty) | **Host-run MCP only — ignored by the dockerized stack** (a host path never resolves inside the `codebadger-mcp` container; do not set it in `.env`). Private key FILE for `ssh://` clones of a custom host. The clone also sets `-o BatchMode=yes`, so a missing key fails fast instead of hanging on a prompt. |
+| `GIT_CLONE_SSH_KNOWN_HOSTS` | `` (empty) | `known_hosts` file pinning the custom servers' host keys (`-o StrictHostKeyChecking=yes`). Unset, the clone falls back to `accept-new`: the key is recorded on first contact, but **in the dockerized stack that record lives in the container and is lost on every recreate**, making it trust-on-first-use each deploy. Like `GIT_CLONE_SSH_COMMAND` this is an in-container path there — put a `known_hosts` in the mounted key dir and set `/keys/known_hosts`. |
+| `GIT_CLONE_SSH_COMMAND` | `` (empty) | Full ssh command override (passed to git as `GIT_SSH_COMMAND` for the clone); takes precedence over both key settings. In the dockerized stack any key path it references must exist **inside the `codebadger-mcp` container** (e.g. `/keys/…`). |
+
+Notes:
+- **Where does the clone run?** In the MCP process, so every path must make
+  sense *there*: host-run MCP → host filesystem; full docker stack → inside
+  the `codebadger-mcp` container. The two key variables above exist because of
+  this split: `GIT_CLONE_SSH_KEY_PATH` is a host path for host-run MCP,
+  while `GIT_CLONE_SSH_KEYS_HOST_DIR` is the compose bridge that maps a host
+  key dir onto the fixed container path `/keys` (hence the container always
+  sees `/keys/id_ed25519`). For a host-run MCP any key file name works
+  (`ssh -i` doesn't care); in the dockerized stack the key **must** be named
+  `id_ed25519` because the in-container path is fixed — or bypass it with
+  `GIT_CLONE_SSH_COMMAND`.
+- Embedded credentials in the `source_path` URL are always rejected. For
+  github.com/gitlab.com private repos pass the PAT via the `github_token`
+  argument; it is injected into the clone URL and stripped from `.git/config`
+  after the clone.
+- `ssh://` URLs may carry a username (`git@…`) but not a password; keys/agent
+  do the auth. scp-style `git@host:path` URLs are not accepted — use
+  `ssh://git@host[:port]/path` (it carries ports unambiguously).
+- **Ports are part of the allowlist.** A bare `forge.lan` entry only permits
+  `ssh://…@forge.lan[:22]/…`, so allowlisting a git server does not also expose
+  every other port on that machine to a caller who can influence `source_path`.
+  Use `forge.lan:3000` for a non-default ssh port, or `forge.lan:*` to accept
+  any port on it.
+- The allowlist still blocks every other host (alternate git hosts, look-alike
+  domains, cloud metadata endpoints, …), so the SSRF posture of
+  [docs/security.md](security.md) is unchanged — you are explicitly trusting
+  the hosts you list.
+
 ### Memory & the Joern pool
 
 Three distinct memory knobs, easy to confuse — keep them straight:

docs/security.md

@@ -73,7 +73,7 @@ The numbered controls are the boundary checks; each is described below.
 | # | Boundary | Control | Where |
 |---|----------|---------|-------|
 | ① | Tool input → MCP | **Allowlist/format validation of every parameter**: `source_type`, `language` (whitelist), `codebase_hash` (`^[a-f0-9]{16}$`), `github_token` & `branch` (anti URL-/arg-injection, e.g. blocks `--upload-pack`), snippet `code`/`filename`/label, regex `pattern` (length + ReDoS shapes). | `src/utils/validators.py` |
-| ①a | Repo URL → clone (**SSRF/undefined-clone prevention**) | **Strict allowlist on remote repos**: only `https://github.com/` or `https://gitlab.com/` (incl. `www.`). Enforced by **two independent gates** — a literal, case-sensitive `https://<host>/` prefix match *and* a parsed-`hostname` allowlist — plus rejection of any non-`https` scheme (`git://`, `ssh://`, `file://`, …), embedded credentials (`user:tok@`), non-default ports, and whitespace/control chars. Blocks userinfo host-smuggling (`https://github.com@evil/…`), internal/metadata hosts, and look-alike domains. | `validators.py` (`validate_repo_url`) |
+| ①a | Repo URL → clone (**SSRF/undefined-clone prevention**) | **Strict allowlist on remote repos**: only `https://github.com/` or `https://gitlab.com/` (incl. `www.`) by default. For the built-in hosts, enforced by **two independent gates** — a literal, case-sensitive `https://<host>/` prefix match *and* a parsed-`hostname` allowlist — plus rejection of any non-`https` scheme (`git://`, `file://`, …), embedded credentials (`user:tok@`), non-default ports, and whitespace/control chars. Blocks userinfo host-smuggling (`https://github.com@evil/…`), internal/metadata hosts, and look-alike domains. **Operator extension**: `GIT_CLONE_EXTRA_HOSTS` (env) explicitly adds `host[:port]` entries which may also be cloned over `ssh://`, gated by the parsed-`hostname` allowlist (the literal-prefix gate is https-only) plus an **exact port match** — a bare entry means port 22, so one allowlisted host is not a licence to reach every port on that machine; `host:*` opts into any. A username but no password is allowed; auth rides in `GIT_SSH_COMMAND`, never the URL. Everything else about the posture (exact-hostname match, no embedded credentials, control chars, ≥`/owner/repo` path) is unchanged; the config is parsed at startup so a typo fails the boot, and the injected `github_token` for github.com/gitlab.com is stripped from `.git/config` after the clone. | `validators.py` (`validate_repo_url`), `services/git_manager.py` |
 | ①b | Snippet code → CPG | **Language validated *and* inferred.** Pasted code is supplied in `<code language="…">` tags (parsed by regex); the declared language must be supported, and a content-signal check **refuses an obviously mislabeled tag** or **ambiguous/undeclared** language — every refusal returns an actionable message rather than building a wrong-language CPG. | `validators.py` (`parse_snippet_blocks`, `validate_and_infer_snippet_language`) |
 | ② | Source staging | **Path confinement + symlink-safe copy.** Local paths must be absolute, are rejected if they contain null bytes/control chars, then `realpath`-canonicalized (collapsing `..` and resolving symlinks *before* any check) and screened against a system-dir denylist (`/etc`, `/proc`, `/sys`, `/root`, …). An optional `ALLOWED_SOURCE_ROOTS` allowlist hard-contains local sources to named roots. Snapshot reads confined with `realpath`+prefix / `commonpath`; the copy never dereferences symlinks whose target escapes the source tree. | `validators.py` (`resolve_host_path`), `core_tools.py` |
 | ②a | Deployment posture | **`CHAT_DEPLOY=true` disables `source_type='local'` entirely** so a chat-facing / multi-tenant MCP cannot read arbitrary host paths — callers must use an allowlisted repo URL or a pasted snippet. | `core_tools.py`, `config.py` |

main.py

@@ -33,7 +33,7 @@
     QueryExecutor,
     CodeBrowsingService
 )
-from src.utils import setup_logging
+from src.utils import setup_logging, validate_extra_repo_hosts_config
 from src.utils import compute_recommendation, current_from_config, render_recommendation
 from src.startup_tuning import apply_startup_tuning, container_mem_limit_mb, parse_mem_to_mb
 from src.health import (
@@ -374,6 +374,11 @@ async def app_lifespan(server: FastMCP):
     )
     logger.info("Starting CodeBadger Server")
 
+    # Fail the boot on a typo'd GIT_CLONE_EXTRA_HOSTS rather than letting it sit
+    # latent until the first ssh:// clone (github/gitlab clones would keep
+    # working, hiding the misconfiguration from the operator).
+    validate_extra_repo_hosts_config()
+
     # Print the memory-aware configuration envelope before the heavy service
     # init, flag drift that risks an OOM cascade, and auto-derive an unset Joern
     # memory budget from host RAM (before the Joern manager is constructed).

src/defaults.py

@@ -66,6 +66,29 @@ def resolve_redis_url() -> str:
 # generate_cpg: a chat-facing MCP must never expose arbitrary host filesystem
 # paths. Callers use a github.com/gitlab.com URL or a pasted snippet instead.
 CHAT_DEPLOY = False
+
+# --- Custom git clone servers (self-hosted Forgejo / Gitea / GitLab, ...) ----
+# Beyond the built-in github.com/gitlab.com https allowlist, an operator can
+# allowlist their own git server(s) for cloning via generate_cpg. Addresses are
+# configured through the environment so a LAN deployment needs no code changes:
+#   GIT_CLONE_EXTRA_HOSTS   ','-separated `host[:port]` entries accepted in
+#                           addition to github.com/gitlab.com. A bare host means
+#                           port 22 only; `host:port` pins that port; `host:*`
+#                           allows any port on it. Custom hosts are cloned over
+#                           ssh:// only (github.com/gitlab.com stay https-only).
+#                           Parsed at startup — a malformed value fails the boot.
+#   GIT_CLONE_SSH_KEY_PATH  Private key for ssh:// clones of custom hosts.
+#   GIT_CLONE_SSH_KNOWN_HOSTS
+#                           known_hosts file pinning the custom servers' host
+#                           keys (StrictHostKeyChecking=yes). Unset = accept-new,
+#                           i.e. trust-on-first-use.
+#   GIT_CLONE_SSH_COMMAND   Full ssh command override (takes precedence over the
+#                           key path; passed to git as GIT_SSH_COMMAND).
+GIT_CLONE_EXTRA_HOSTS = ""
+GIT_CLONE_SSH_KEY_PATH = ""
+GIT_CLONE_SSH_KNOWN_HOSTS = ""
+GIT_CLONE_SSH_COMMAND = ""
+
 # Optional ':'-separated allowlist of host directory roots that source_type=
 # 'local' paths must canonically resolve within. Empty = no allowlist (the
 # denylist + symlink-resolving canonicalization in resolve_host_path still apply).