From 1a04a16fd1f19ca18f1d38c178c0f71f239f77fe Mon Sep 17 00:00:00 2001 From: Rtosshy Date: Sun, 2 Aug 2026 12:24:37 +0000 Subject: [PATCH 1/2] fix(devcontainer): start dockerd and fix cache permissions on container start postAttachCommand only fires when an editor client attaches, so sessions opened with `gh codespace ssh` never ran setup.sh. That left the container without a Docker daemon and with root-owned cache volumes, which surfaced much later as unrelated-looking failures: Cannot connect to the Docker daemon at unix:///var/run/docker.sock open /go/pkg/sumdb/sum.golang.org/latest: no such file or directory error Error: EACCES: permission denied, mkdir '~/.yarn/cache/v6' Move both concerns out of setup.sh into scripts that postStartCommand runs at every container start, one concern each. setup.sh still calls them -- they are preconditions for its own work (yarn/go write into those volumes, cleanup_docker_if_needed shells out to docker) and both are no-ops once satisfied. That call also moves to the top of main(), so the daemon is up before cleanup_docker_if_needed uses it. Starting the daemon via `service docker start` keeps its log at /var/log/docker.log; the previous `nohup sudo dockerd > /tmp/dockerd.log` left nothing behind once /tmp was cleared. Co-Authored-By: Claude Opus 5 --- .devcontainer/devcontainer.json | 11 +++-- .devcontainer/fix-permissions.sh | 58 ++++++++++++++++++++++++++ .devcontainer/setup.sh | 70 ++++++++------------------------ .devcontainer/start-docker.sh | 32 +++++++++++++++ 4 files changed, 116 insertions(+), 55 deletions(-) create mode 100755 .devcontainer/fix-permissions.sh create mode 100755 .devcontainer/start-docker.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1a43d13df0..cdf35fc240 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -76,8 +76,13 @@ "memory": "8gb", "storage": "32gb" }, - // Start sshd at every container start so `gh codespace ssh` works without an editor attach - // (postAttachCommand only fires when an editor client attaches) - "postStartCommand": "bash .devcontainer/start-sshd.sh", + // Everything a container start must do, regardless of how the user connects. + // postAttachCommand (setup.sh) only fires when an editor client attaches, so + // `gh codespace ssh` sessions used to get no sshd, no Docker daemon and + // root-owned cache volumes. Each script owns one concern: + // start-sshd.sh - sshd, so `gh codespace ssh` can connect + // fix-permissions.sh - chown the mounted cache volumes to the container user + // start-docker.sh - start dockerd + "postStartCommand": "bash .devcontainer/start-sshd.sh && bash .devcontainer/fix-permissions.sh && bash .devcontainer/start-docker.sh", "postAttachCommand": "bash .devcontainer/setup.sh" } diff --git a/.devcontainer/fix-permissions.sh b/.devcontainer/fix-permissions.sh new file mode 100755 index 0000000000..19e87f95e7 --- /dev/null +++ b/.devcontainer/fix-permissions.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Makes the mounted cache volumes writable by the container user. +# +# devcontainer.json mounts several named Docker volumes (Go module cache, Go +# tools, node_modules, yarn cache, minikube cache). A named volume that is empty +# on first use is created root-owned, so every tool that writes into one fails +# with EACCES until the ownership is fixed: +# +# yarn install -> Error: EACCES: permission denied, mkdir '~/.yarn/cache/v6' +# go build -> open /go/pkg/sumdb/sum.golang.org/latest: no such file or directory +# +# This used to run only from setup.sh (postAttachCommand), which fires solely +# when an editor attaches -- `gh codespace ssh` sessions hit the failures above. +# It is wired to postStartCommand instead so every container start is covered. +# +# Kept separate from start-docker.sh on purpose: this script only ever chowns +# paths, that one only ever deals with the Docker daemon. +set -e + +if id codespace &> /dev/null; then + USER_NAME=codespace +else + USER_NAME=$(whoami) +fi +HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6) + +# Directories that must exist and be owned by the user. `go` cannot create +# /go/pkg/sumdb itself because /go/pkg is root-owned, so mkdir -p comes first. +OWNED_DIRS=( + /go/pkg/mod + /go/pkg/sumdb + "$HOME_DIR/go-tools" + "$HOME_DIR/.yarn" + "$HOME_DIR/.minikube" + /workspaces/bucketeer/ui/dashboard/node_modules + /workspaces/bucketeer/evaluation/typescript/node_modules +) + +fixed=0 +for dir in "${OWNED_DIRS[@]}"; do + sudo mkdir -p "$dir" + # Skip the recursive chown when the top-level owner is already correct -- + # these trees hold tens of thousands of files and this runs on every start. + if [ "$(stat -c '%U' "$dir")" != "$USER_NAME" ]; then + echo "🔑 Fixing ownership of $dir..." + sudo chown -R "$USER_NAME:$USER_NAME" "$dir" + fixed=$((fixed + 1)) + fi +done + +# minikube refuses to start if its cache is not user-writable. +chmod -R u+wrx "$HOME_DIR/.minikube" 2> /dev/null || true + +if [ "$fixed" -eq 0 ]; then + echo "✅ Cache volume permissions already correct" +else + echo "✅ Fixed permissions on $fixed cache volume(s)" +fi diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 42a21907bf..497fe3da80 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -92,46 +92,23 @@ if command -v kubectl &> /dev/null; then fi # Function to fix permissions on cache directories +# The actual work lives in fix-permissions.sh, which postStartCommand also runs +# so that `gh codespace ssh` sessions (no editor attach, no postAttachCommand) +# get writable cache volumes too. Kept as a call here so an editor attach still +# repairs anything that changed since the container started. fix_cache_permissions() { print_status "Ensuring cache directories have correct permissions..." + bash "$(dirname "${BASH_SOURCE[0]}")/fix-permissions.sh" +} - # Check if codespace user exists - if id "codespace" &>/dev/null; then - USER_NAME="codespace" - else - USER_NAME=$(whoami) - print_status "codespace user not found, using current user: $USER_NAME" - fi - - # Fix Go modules cache permissions - sudo mkdir -p /go/pkg/mod /go/pkg/sumdb - sudo chown -R $USER_NAME:$USER_NAME /go/pkg/mod /go/pkg/sumdb - - # Fix Go tools directory permissions - if [ -d "/home/$USER_NAME/go-tools" ]; then - sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/go-tools - fi - - # Fix Yarn cache permissions - if [ -d "/home/$USER_NAME/.yarn" ]; then - sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/.yarn - fi - - # Fix node_modules permissions - for project in "${NODE_PROJECTS[@]}"; do - if [ -d "$project/node_modules" ]; then - sudo chown -R $USER_NAME:$USER_NAME "$project/node_modules" - fi - done - - # Fix minikube cache permissions - if [ -d "/home/$USER_NAME/.minikube" ]; then - sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/.minikube - chmod -R u+wrx /home/$USER_NAME/.minikube - print_status "Fixed minikube cache permissions" - fi - - print_success "Cache permissions fixed" +# Function to make sure the Docker daemon is up +# Same delegation as above: postStartCommand runs start-docker.sh at every +# container start, so this is normally a no-op. It stays here because +# cleanup_docker_if_needed below shells out to `docker`, and because +# postStartCommand does not re-run when an editor re-attaches to a container +# whose daemon died mid-session. +ensure_docker_running() { + bash "$(dirname "${BASH_SOURCE[0]}")/start-docker.sh" } # Function to check if Go tools are installed @@ -417,8 +394,11 @@ cleanup_docker_if_needed() { # Main setup logic main() { - # Fix any permission issues with mounted cache volumes first + # Preconditions for everything below: yarn/go write into the mounted cache + # volumes, and cleanup_docker_if_needed shells out to `docker`. Both must be + # satisfied before any of the install steps run. fix_cache_permissions + ensure_docker_running print_status "Checking cache status..." @@ -503,17 +483,3 @@ main() { # Run main function main "$@" -# Start Docker daemon automatically after main setup is complete -if ! docker info > /dev/null 2>&1; then - echo "🐳 Starting Docker daemon..." - nohup sudo dockerd > /tmp/dockerd.log 2>&1 < /dev/null & - # Wait for Docker to be ready - while ! docker info > /dev/null 2>&1; do - sleep 1 - done - echo "✅ Docker daemon started successfully" - # Reset cursor position for clean terminal state - printf "\r" -else - echo "✅ Docker daemon already running" -fi diff --git a/.devcontainer/start-docker.sh b/.devcontainer/start-docker.sh new file mode 100755 index 0000000000..3459a42ce6 --- /dev/null +++ b/.devcontainer/start-docker.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Starts the Docker daemon for this dev container. +# +# This used to live at the end of setup.sh (postAttachCommand), which fires +# solely when an editor client attaches -- `gh codespace ssh` sessions ended up +# with no Docker daemon at all, so minikube and every `make` target that talks +# to Docker failed with "Cannot connect to the Docker daemon". +# +# Kept separate from fix-permissions.sh on purpose: that script only ever chowns +# paths, this one only ever deals with the Docker daemon. +set -e + +if docker info > /dev/null 2>&1; then + echo "✅ Docker daemon already running" + exit 0 +fi + +echo "🐳 Starting Docker daemon..." +# `service` keeps the daemon log at /var/log/docker.log. The previous +# `nohup sudo dockerd > /tmp/dockerd.log` left no trace once /tmp was cleared. +sudo service docker start > /dev/null + +for _ in $(seq 1 60); do + if docker info > /dev/null 2>&1; then + echo "✅ Docker daemon started successfully" + exit 0 + fi + sleep 1 +done + +echo "❌ Docker daemon did not become ready; see /var/log/docker.log" >&2 +exit 1 From 36ee64be82eca79e2ab46e3cf8c45c6ccc41b996 Mon Sep 17 00:00:00 2001 From: Rtosshy Date: Sun, 2 Aug 2026 12:25:12 +0000 Subject: [PATCH 2/2] fix(devcontainer): align iptables backend with the Codespaces host daemon Codespaces runs the dev container in the host network namespace, so the Codespaces VM's Docker daemon and our nested one write firewall rules into the same namespace. The outer daemon is an older release on iptables-legacy: it sets `FORWARD policy DROP` there and opens holes only for its own docker0, which nothing is even attached to under host networking. Our daemon defaults to the nft backend, so minikube's bridge got its ACCEPT rules in a different table from the DROP and every forwarded packet was dropped. The result is a cluster that starts fine from preloaded images but cannot reach anything: Failed to connect to registry.k8s.io port 443: Connection timed out Back-off pulling image "registry.k8s.io/ingress-nginx/kube-webhook-certgen" dial tcp 10.96.0.1:443: i/o timeout DNS still resolves throughout, because the node's resolver is the bridge gateway -- an INPUT path that never touches FORWARD. That asymmetry makes this look like a TLS or registry problem rather than a host firewall one. Pick the legacy backend when `ip_tables` is loaded, which is the signal that something in this namespace already uses it. The nested daemon then writes its ACCEPT rules into the same table as the DROP policy and they take effect before it. This is what the upstream docker-in-docker devcontainer feature does; we cannot use that feature because Docker is installed directly in the image. On a host using nft the condition does not match and nothing changes. Co-Authored-By: Claude Opus 5 --- .devcontainer/devcontainer.json | 5 ++++- .devcontainer/start-docker.sh | 37 ++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cdf35fc240..5c4f85b816 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -82,7 +82,10 @@ // root-owned cache volumes. Each script owns one concern: // start-sshd.sh - sshd, so `gh codespace ssh` can connect // fix-permissions.sh - chown the mounted cache volumes to the container user - // start-docker.sh - start dockerd + // start-docker.sh - align the iptables backend with the Codespaces host + // daemon, then start dockerd (the alignment must happen + // before dockerd starts or the nested daemon has no + // network at all) "postStartCommand": "bash .devcontainer/start-sshd.sh && bash .devcontainer/fix-permissions.sh && bash .devcontainer/start-docker.sh", "postAttachCommand": "bash .devcontainer/setup.sh" } diff --git a/.devcontainer/start-docker.sh b/.devcontainer/start-docker.sh index 3459a42ce6..ed87b2f6f6 100755 --- a/.devcontainer/start-docker.sh +++ b/.devcontainer/start-docker.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Starts the Docker daemon for this dev container. +# Starts the Docker daemon for this dev container, on a firewall it can use. # # This used to live at the end of setup.sh (postAttachCommand), which fires # solely when an editor client attaches -- `gh codespace ssh` sessions ended up @@ -10,6 +10,41 @@ # paths, this one only ever deals with the Docker daemon. set -e +# --- Align the iptables backend with the outer daemon ------------------------ +# Codespaces runs this container in the *host* network namespace, so the +# Codespaces VM's own Docker daemon and the daemon we start here write firewall +# rules into the same namespace. That outer daemon is an older release using +# iptables-legacy: it sets `FORWARD policy DROP` and only opens holes for its +# own docker0. If we then use the nft backend, our bridges (minikube's included) +# never get an ACCEPT rule in the table that holds the DROP, so every forwarded +# packet dies there -- image pulls from inside the cluster time out while the +# host itself reaches the registry fine, and nothing in the minikube logs points +# at the host firewall. +# +# Matching the backend makes our daemon add its own ACCEPT rules right next to +# the DROP policy, which resolves it. This is the same fix the upstream +# docker-in-docker devcontainer feature applies, for the same reason; we cannot +# use that feature here because Docker is installed directly in the image. +# +# `ip_tables` being loaded is the signal that something in this namespace +# already uses the legacy backend. The /sys/module check covers kernels that +# build it in rather than as a module. On a host that uses nft, none of this +# matches and the block is a no-op. +if type iptables-legacy > /dev/null 2>&1 \ + && { grep -qE '^ip_tables\b' /proc/modules || [ -d /sys/module/ip_tables ]; } \ + && update-alternatives --list iptables 2>/dev/null | grep -q '/usr/sbin/iptables-legacy'; then + + current=$(update-alternatives --query iptables 2>/dev/null | awk '/^Value:/ {print $2}') + if [ "$current" != "/usr/sbin/iptables-legacy" ]; then + echo "🔧 Switching iptables backend to legacy to match the host daemon..." + sudo update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null + sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null 2>&1 || true + fi +fi + +# --- Start dockerd ----------------------------------------------------------- +# Order matters: the switch above only affects a daemon started afterwards. A +# running dockerd has already written its rules to whichever table it picked. if docker info > /dev/null 2>&1; then echo "✅ Docker daemon already running" exit 0