fix: devcontainer startup for ssh sessions

.devcontainer/devcontainer.json

@@ -76,8 +76,16 @@
     "memory": "8gb",
     "storage": "32gb"
   },
-  // Start sshd at every container start so `gh codespace ssh` works without an editor attach
-  // (postAttachCommand only fires when an editor client attaches)
-  "postStartCommand": "bash .devcontainer/start-sshd.sh",
+  // Everything a container start must do, regardless of how the user connects.
+  // postAttachCommand (setup.sh) only fires when an editor client attaches, so
+  // `gh codespace ssh` sessions used to get no sshd, no Docker daemon and
+  // root-owned cache volumes. Each script owns one concern:
+  //   start-sshd.sh      - sshd, so `gh codespace ssh` can connect
+  //   fix-permissions.sh - chown the mounted cache volumes to the container user
+  //   start-docker.sh    - align the iptables backend with the Codespaces host
+  //                        daemon, then start dockerd (the alignment must happen
+  //                        before dockerd starts or the nested daemon has no
+  //                        network at all)
+  "postStartCommand": "bash .devcontainer/start-sshd.sh && bash .devcontainer/fix-permissions.sh && bash .devcontainer/start-docker.sh",
   "postAttachCommand": "bash .devcontainer/setup.sh"
 }

.devcontainer/fix-permissions.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Makes the mounted cache volumes writable by the container user.
+#
+# devcontainer.json mounts several named Docker volumes (Go module cache, Go
+# tools, node_modules, yarn cache, minikube cache). A named volume that is empty
+# on first use is created root-owned, so every tool that writes into one fails
+# with EACCES until the ownership is fixed:
+#
+#   yarn install  -> Error: EACCES: permission denied, mkdir '~/.yarn/cache/v6'
+#   go build      -> open /go/pkg/sumdb/sum.golang.org/latest: no such file or directory
+#
+# This used to run only from setup.sh (postAttachCommand), which fires solely
+# when an editor attaches -- `gh codespace ssh` sessions hit the failures above.
+# It is wired to postStartCommand instead so every container start is covered.
+#
+# Kept separate from start-docker.sh on purpose: this script only ever chowns
+# paths, that one only ever deals with the Docker daemon.
+set -e
+
+if id codespace &> /dev/null; then
+    USER_NAME=codespace
+else
+    USER_NAME=$(whoami)
+fi
+HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6)
+
+# Directories that must exist and be owned by the user. `go` cannot create
+# /go/pkg/sumdb itself because /go/pkg is root-owned, so mkdir -p comes first.
+OWNED_DIRS=(
+    /go/pkg/mod
+    /go/pkg/sumdb
+    "$HOME_DIR/go-tools"
+    "$HOME_DIR/.yarn"
+    "$HOME_DIR/.minikube"
+    /workspaces/bucketeer/ui/dashboard/node_modules
+    /workspaces/bucketeer/evaluation/typescript/node_modules
+)
+
+fixed=0
+for dir in "${OWNED_DIRS[@]}"; do
+    sudo mkdir -p "$dir"
+    # Skip the recursive chown when the top-level owner is already correct --
+    # these trees hold tens of thousands of files and this runs on every start.
+    if [ "$(stat -c '%U' "$dir")" != "$USER_NAME" ]; then
+        echo "🔑 Fixing ownership of $dir..."
+        sudo chown -R "$USER_NAME:$USER_NAME" "$dir"
+        fixed=$((fixed + 1))
+    fi
+done
+
+# minikube refuses to start if its cache is not user-writable.
+chmod -R u+wrx "$HOME_DIR/.minikube" 2> /dev/null || true
+
+if [ "$fixed" -eq 0 ]; then
+    echo "✅ Cache volume permissions already correct"
+else
+    echo "✅ Fixed permissions on $fixed cache volume(s)"
+fi

.devcontainer/setup.sh

@@ -92,46 +92,23 @@ if command -v kubectl &> /dev/null; then
 fi
 
 # Function to fix permissions on cache directories
+# The actual work lives in fix-permissions.sh, which postStartCommand also runs
+# so that `gh codespace ssh` sessions (no editor attach, no postAttachCommand)
+# get writable cache volumes too. Kept as a call here so an editor attach still
+# repairs anything that changed since the container started.
 fix_cache_permissions() {
     print_status "Ensuring cache directories have correct permissions..."
+    bash "$(dirname "${BASH_SOURCE[0]}")/fix-permissions.sh"
+}
 
-    # Check if codespace user exists
-    if id "codespace" &>/dev/null; then
-        USER_NAME="codespace"
-    else
-        USER_NAME=$(whoami)
-        print_status "codespace user not found, using current user: $USER_NAME"
-    fi
-
-    # Fix Go modules cache permissions
-    sudo mkdir -p /go/pkg/mod /go/pkg/sumdb
-    sudo chown -R $USER_NAME:$USER_NAME /go/pkg/mod /go/pkg/sumdb
-
-    # Fix Go tools directory permissions
-    if [ -d "/home/$USER_NAME/go-tools" ]; then
-        sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/go-tools
-    fi
-
-    # Fix Yarn cache permissions
-    if [ -d "/home/$USER_NAME/.yarn" ]; then
-        sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/.yarn
-    fi
-
-    # Fix node_modules permissions
-    for project in "${NODE_PROJECTS[@]}"; do
-        if [ -d "$project/node_modules" ]; then
-            sudo chown -R $USER_NAME:$USER_NAME "$project/node_modules"
-        fi
-    done
-
-    # Fix minikube cache permissions
-    if [ -d "/home/$USER_NAME/.minikube" ]; then
-        sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/.minikube
-        chmod -R u+wrx /home/$USER_NAME/.minikube
-        print_status "Fixed minikube cache permissions"
-    fi
-
-    print_success "Cache permissions fixed"
+# Function to make sure the Docker daemon is up
+# Same delegation as above: postStartCommand runs start-docker.sh at every
+# container start, so this is normally a no-op. It stays here because
+# cleanup_docker_if_needed below shells out to `docker`, and because
+# postStartCommand does not re-run when an editor re-attaches to a container
+# whose daemon died mid-session.
+ensure_docker_running() {
+    bash "$(dirname "${BASH_SOURCE[0]}")/start-docker.sh"
 }
 
 # Function to check if Go tools are installed
@@ -417,8 +394,11 @@ cleanup_docker_if_needed() {
 
 # Main setup logic
 main() {
-    # Fix any permission issues with mounted cache volumes first
+    # Preconditions for everything below: yarn/go write into the mounted cache
+    # volumes, and cleanup_docker_if_needed shells out to `docker`. Both must be
+    # satisfied before any of the install steps run.
     fix_cache_permissions
+    ensure_docker_running
 
     print_status "Checking cache status..."
 
@@ -503,17 +483,3 @@ main() {
 # Run main function
 main "$@"
 
-# Start Docker daemon automatically after main setup is complete
-if ! docker info > /dev/null 2>&1; then
-    echo "🐳 Starting Docker daemon..."
-    nohup sudo dockerd > /tmp/dockerd.log 2>&1 < /dev/null &
-    # Wait for Docker to be ready
-    while ! docker info > /dev/null 2>&1; do
-        sleep 1
-    done
-    echo "✅ Docker daemon started successfully"
-    # Reset cursor position for clean terminal state
-    printf "\r"
-else
-    echo "✅ Docker daemon already running"
-fi

.devcontainer/start-docker.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Starts the Docker daemon for this dev container, on a firewall it can use.
+#
+# This used to live at the end of setup.sh (postAttachCommand), which fires
+# solely when an editor client attaches -- `gh codespace ssh` sessions ended up
+# with no Docker daemon at all, so minikube and every `make` target that talks
+# to Docker failed with "Cannot connect to the Docker daemon".
+#
+# Kept separate from fix-permissions.sh on purpose: that script only ever chowns
+# paths, this one only ever deals with the Docker daemon.
+set -e
+
+# --- Align the iptables backend with the outer daemon ------------------------
+# Codespaces runs this container in the *host* network namespace, so the
+# Codespaces VM's own Docker daemon and the daemon we start here write firewall
+# rules into the same namespace. That outer daemon is an older release using
+# iptables-legacy: it sets `FORWARD policy DROP` and only opens holes for its
+# own docker0. If we then use the nft backend, our bridges (minikube's included)
+# never get an ACCEPT rule in the table that holds the DROP, so every forwarded
+# packet dies there -- image pulls from inside the cluster time out while the
+# host itself reaches the registry fine, and nothing in the minikube logs points
+# at the host firewall.
+#
+# Matching the backend makes our daemon add its own ACCEPT rules right next to
+# the DROP policy, which resolves it. This is the same fix the upstream
+# docker-in-docker devcontainer feature applies, for the same reason; we cannot
+# use that feature here because Docker is installed directly in the image.
+#
+# `ip_tables` being loaded is the signal that something in this namespace
+# already uses the legacy backend. The /sys/module check covers kernels that
+# build it in rather than as a module. On a host that uses nft, none of this
+# matches and the block is a no-op.
+if type iptables-legacy > /dev/null 2>&1 \
+    && { grep -qE '^ip_tables\b' /proc/modules || [ -d /sys/module/ip_tables ]; } \
+    && update-alternatives --list iptables 2>/dev/null | grep -q '/usr/sbin/iptables-legacy'; then
+
+    current=$(update-alternatives --query iptables 2>/dev/null | awk '/^Value:/ {print $2}')
+    if [ "$current" != "/usr/sbin/iptables-legacy" ]; then
+        echo "🔧 Switching iptables backend to legacy to match the host daemon..."
+        sudo update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null
+        sudo update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null 2>&1 || true
+    fi
+fi
+
+# --- Start dockerd -----------------------------------------------------------
+# Order matters: the switch above only affects a daemon started afterwards. A
+# running dockerd has already written its rules to whichever table it picked.
+if docker info > /dev/null 2>&1; then
+    echo "✅ Docker daemon already running"
+    exit 0
+fi
+
+echo "🐳 Starting Docker daemon..."
+# `service` keeps the daemon log at /var/log/docker.log. The previous
+# `nohup sudo dockerd > /tmp/dockerd.log` left no trace once /tmp was cleared.
+sudo service docker start > /dev/null
+
+for _ in $(seq 1 60); do
+    if docker info > /dev/null 2>&1; then
+        echo "✅ Docker daemon started successfully"
+        exit 0
+    fi
+    sleep 1
+done
+
+echo "❌ Docker daemon did not become ready; see /var/log/docker.log" >&2
+exit 1