bootstrap.sh

#!/bin/bash
#
# Easy Bootstrap Script for Single-Node Setup
#
# This script simplifies the process of bootstrapping the system on a single
# server for development or as the initial control node for a new cluster.
# It runs the main Ansible playbook using a local inventory file.

# --- Colors ---
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color
BOLD='\033[1m'

# --- Help Menu ---
show_help() {
    echo "Usage: $0 [options]"
    echo ""
    echo "This script runs the main Ansible playbook to bootstrap the system."
    echo ""
    echo "Options:"
    echo "  --role <role>                Specify the role for this node (all, controller, worker). Default: all."
    echo "  --controller-ip <ip>         Required if --role is 'worker'. IP address of the controller node."
    echo "  --tags <tags>                Comma-separated list of Ansible tags to run."
    echo "  --user <user>                Specify the target user for Ansible. Default: pipecatapp."
    echo "  --github-ssh-user <user>     Specify a GitHub username to import SSH keys for. Can be used multiple times."
    echo "  --purge-jobs                 Stop and purge all running Nomad jobs."
    echo "  --clean-git                  Clean the repository of all untracked files (interactive prompt)."
    echo "  --system-cleanup             Perform a full system cleanup (Purge Jobs, Clean System, Clean Git), with interactive prompts."
    echo "  --verbose [level]            Set verbosity level (0-4). Default 0, or 3 if flag is used without value."
    echo "  --debug                      Alias for --verbose 4."
    echo "  --leave-services-running     Do not clean up Nomad and Consul data on startup."
    echo "  --external-model-server      Skip large model downloads and builds, assuming an external server."
    echo "  --tier <tier>                Specify the node tier (edge, mid, core). Default: mid."
    echo "  --deploy-full-stack          Deploy the full application stack (AI agents, models) instead of just infrastructure."
    echo "  --deploy-partial-stack       Deploy a partial application stack (e.g. 4-8B models) for mid-tier worker nodes."
    echo "  --deploy-minimal-stack       Deploy a minimal application stack (e.g. audio, kiosk, status) for low resource nodes."
    echo "  --test-mode                  Test mode: Purges application jobs after they are deployed to save memory during testing."
    echo "  --continue                   Resume from the last successfully completed playbook."
    echo "  --benchmark                  Run benchmark tests."
    echo "  --deploy-docker              Deploy the pipecat application using Docker (Default)."
    echo "  --dry-run                    Perform a dry run to validate playbooks without applying changes."
    echo "  --skip-setup                 Skip running the initial machine setup script (initial-setup/setup.sh)."
    echo "  --run-local                  Deploy the pipecat application using local raw_exec (for debugging)."
    echo "  --home-assistant-debug       Enable debug mode for Home Assistant."
    echo "  --container                  Run the entire infrastructure inside a single large container."
    echo "  --watch <target>             Pause for inspection after the specified target (task/role) completes."
    echo "  --status                     Show the current cluster and Opencode status and exit."
    echo "  -h, --help                   Display this help message and exit."
    echo ""
    echo "OS Recovery Snapshot Options:"
    echo "  --create-snapshot            Create a Btrfs pre-deployment snapshot of the environment."
    echo "  --list-snapshots             List existing Btrfs pre-deployment snapshots."
    echo "  --rollback-snapshot [snap]   Rollback to a specified pre-deployment snapshot (or latest)."
    echo ""
    echo "Cluster and Node Recovery Options:"
    echo "  --heal-cluster               Run the cluster healing playbook to restore core services."
    echo "  --troubleshoot [args]        Run the unified system-wide troubleshoot/healing utility."
    echo "  --recover-node <ip>          Attempt to recover a remote node by its IP address."
    echo "  --ipmi-host <host>           IPMI network address (BMC IP) for remote node recovery."
    echo "  --ipmi-user <user>           IPMI username for remote node recovery."
    echo "  --ipmi-password <pass>       IPMI password for remote node recovery."
    echo "  --force-pxe                  Force the node to boot into PXE on reset during recovery."
    echo "  --pxe-server-ip <ip>         IP address of the PXE server to check before forcing PXE."
    echo ""
    echo "SmolAgent Autonomous Recovery Options:"
    echo "  If the script crashes, it will attempt to use the SmolAgent to diagnose and fix the error."
    echo "  These can also be set via environment variables (AGENT_API_BASE, AGENT_MODEL, AGENT_API_KEY)."
    echo "  --agent-api-base <url>       URL for a custom LLM provider, like a local Ollama instance (e.g. http://192.168.1.100:11434/v1)."
    echo "  --agent-model <model>        The LLM model to use for debugging (e.g. qwen3:14b, llama3.2:3b, mistral, qwen2.5:1.5b)."
    echo "  --agent-api-key <key>        API key for the model provider (a dummy key like 'sk-dummy' can be used for Ollama)."
    echo "  -y, --yes                    Automatically answer 'yes' to all interactive prompts (useful for --system-cleanup)."
}

# --- Initialize flags ---
USE_CONTAINER=false
DO_CLEAN_GIT=false
DO_SYSTEM_CLEANUP=false
AUTO_YES=false
DO_PURGE_JOBS=false
DO_STATUS=false
DO_DRY_RUN=false
DO_SKIP_SETUP=false
DO_HEAL_CLUSTER=false
DO_RECOVER_NODE=false
VERBOSE_LEVEL=0
ROLE=""
CONTROLLER_IP=""
RECOVER_NODE_IP=""
DO_CREATE_SNAPSHOT=false
DO_LIST_SNAPSHOTS=false
DO_ROLLBACK_SNAPSHOT=false
ROLLBACK_SNAPSHOT_TARGET=""
IPMI_HOST=""
IPMI_USER=""
IPMI_PASSWORD=""
FORCE_PXE=false
PXE_SERVER_IP=""
CLI_AGENT_API_BASE=""
CLI_AGENT_MODEL=""
CLI_AGENT_API_KEY=""
DO_TROUBLESHOOT=false
declare -a TROUBLESHOOT_ARGS

# --- Network Discovery ---
find_controller() {
    echo -e "\n${BOLD}=== Network Discovery ===${NC}"
    echo -e "Searching for an existing controller on the network..."

    # 1. Provisioning Underlay
    local PXE_SUBNET
    PXE_SUBNET=$(grep -oP '^pxe_subnet:\s*"\K[^"]+' group_vars/all.yaml 2>/dev/null || echo "10.0.0.0")
    if [[ ! "$PXE_SUBNET" == */* ]]; then
        PXE_SUBNET="${PXE_SUBNET}/24"
    fi

    # 2. Cluster Overlay
    local OVERLAY_SUBNET
    OVERLAY_SUBNET=$(grep -oP '^overlay_subnet:\s*"\K[^"]+' group_vars/all.yaml 2>/dev/null || echo "100.64.0.0/10")

    # 3. Local Host Network / Fallback
    local LOCAL_SUBNET
    LOCAL_SUBNET=$(ip route show default | awk '/default/ {print $3}' | awk -F. '{print $1"."$2"."$3".0/24"}' 2>/dev/null || echo "192.168.1.0/24")

    # Install nmap if missing
    if ! command -v nmap >/dev/null 2>&1; then
        if [ "$DO_DRY_RUN" = true ]; then
            echo -e "${RED}❌ Dry run failed: nmap is missing. Please install it manually before running a dry run.${NC}"
            exit 1
        fi
        echo -e "⏳ Installing nmap for network scanning..."
        if sudo -n true 2>/dev/null; then
            sudo apt-get update -qq && sudo apt-get install -y nmap -qq >/dev/null 2>&1
        else
            echo -e "${YELLOW}⚠️  Requires nmap. You may be prompted for your sudo password to install it.${NC}"
            sudo apt-get update -qq && sudo apt-get install -y nmap -qq >/dev/null 2>&1
        fi
    fi

    local SCAN_RESULTS=""

    # Add a fast-path scan for the first /24 block of the overlay to save time
    local OVERLAY_FAST_PATH="100.64.0.0/24"
    # Only add OVERLAY_FAST_PATH to array if it is different from OVERLAY_SUBNET to avoid scanning same thing twice
    local SUBNETS_TO_SCAN=("$PXE_SUBNET" "$LOCAL_SUBNET" "$OVERLAY_FAST_PATH")

    spinner() {
        local pid=$1
        local delay=0.1
        local spinstr="|\/-\\"
        while kill -0 "$pid" 2>/dev/null; do
            local temp=${spinstr#?}
            printf " [%c]  " "$spinstr"
            local spinstr=$temp${spinstr%"$temp"}
            sleep $delay
            printf ""
        done
        printf "    "
    }

    echo -e "Scanning networks on port 4646 (Nomad):"

    for subnet in "${SUBNETS_TO_SCAN[@]}"; do
        echo -n -e "  - ${CYAN}${subnet}${NC}... "

        # Fast scan for /10 overlay limits max-retries and timeout aggressively
        # shellcheck disable=SC2086
        local nmap_args="-n -p 4646 --open -T5 --max-retries 1 --host-timeout 500ms --min-rate 10000"

        local tmp_file
        tmp_file=$(mktemp)

        # Run nmap in the background so we can show a spinner
        # shellcheck disable=SC2086
        nmap $nmap_args -oG - "$subnet" > "$tmp_file" 2>/dev/null &
        local nmap_pid=$!

        # Ensure we kill the background process and delete temp file if interrupted
        # Note: escaping the command text for the python replacement
        trap 'kill $nmap_pid 2>/dev/null; rm -f "$tmp_file"; exit 1' INT TERM

        spinner "$nmap_pid"

        # Reset trap back to script default
        trap - INT TERM

        SCAN_RESULTS=$(awk '/4646\/open/ {print $2}' "$tmp_file" 2>/dev/null)
        rm -f "$tmp_file"

        if [ -n "$SCAN_RESULTS" ]; then
            echo -e "\r  - ${CYAN}${subnet}${NC}... Done.          "
            CONTROLLER_IP=$(echo "$SCAN_RESULTS" | head -n 1)
            echo -e "${GREEN}✅ Found controller at: ${CONTROLLER_IP} (on ${subnet})${NC}"
            return 0
        fi
        echo -e "\r  - ${CYAN}${subnet}${NC}... Done.          "
    done

    echo -e "${YELLOW}⚠️  No controller found on any scanned networks.${NC}"
    return 1
}

# --- Profile System Resources ---
profile_system() {
    echo -e "\n${BOLD}=== Profiling System Resources ===${NC}"

    local CPU_CORES
    CPU_CORES=$(nproc 2>/dev/null || echo 1)

    # 1. RAM Calculation
    local TOTAL_RAM_KB
    TOTAL_RAM_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)
    local RAW_AVAILABLE_RAM_KB
    RAW_AVAILABLE_RAM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)

    local PRE_ALLOCATED_RAM_KB=0
    local CLUSTER_SERVICES=("nomad" "consul" "dockerd" "docker" "containerd" "pipecatapp")
    for svc in "${CLUSTER_SERVICES[@]}"; do
        # sum of rss for processes matching name
        local svc_ram
        svc_ram=$(ps -C "$svc" -o rss= 2>/dev/null | awk '{sum+=$1} END {print sum}')
        if [[ -n "$svc_ram" && "$svc_ram" =~ ^[0-9]+$ ]]; then
            PRE_ALLOCATED_RAM_KB=$((PRE_ALLOCATED_RAM_KB + svc_ram))
        fi
    done

    # Add KBs together first to avoid rounding to 0 GB
    local TOTAL_RAM_GB=$(( TOTAL_RAM_KB / 1024 / 1024 ))
    local PRE_ALLOCATED_RAM_GB=$(( PRE_ALLOCATED_RAM_KB / 1024 / 1024 ))
    local EFFECTIVE_AVAILABLE_RAM_KB=$(( RAW_AVAILABLE_RAM_KB + PRE_ALLOCATED_RAM_KB ))
    local EFFECTIVE_AVAILABLE_RAM_GB=$(( EFFECTIVE_AVAILABLE_RAM_KB / 1024 / 1024 ))

    # 2. Disk Calculation
    local RAW_DISK_KB
    RAW_DISK_KB=$(df -k / | awk 'NR==2 {print $4}' 2>/dev/null || echo 0)

    local PRE_ALLOCATED_DISK_KB=0
    local CLUSTER_DIRS=("/opt/nomad" "/var/lib/docker" "/opt/pipecatapp")
    for dir in "${CLUSTER_DIRS[@]}"; do
        if [ -d "$dir" ]; then
            local dir_size
            dir_size=$(sudo -n du -sk "$dir" 2>/dev/null | awk '{print $1}')
            if [[ -n "$dir_size" && "$dir_size" =~ ^[0-9]+$ ]]; then
                PRE_ALLOCATED_DISK_KB=$((PRE_ALLOCATED_DISK_KB + dir_size))
            fi
        fi
    done

    local RAW_DISK_GB=$(( RAW_DISK_KB / 1024 / 1024 ))
    local PRE_ALLOCATED_DISK_GB=$(( PRE_ALLOCATED_DISK_KB / 1024 / 1024 ))
    local EFFECTIVE_DISK_KB=$(( RAW_DISK_KB + PRE_ALLOCATED_DISK_KB ))
    local EFFECTIVE_DISK_GB=$(( EFFECTIVE_DISK_KB / 1024 / 1024 ))

    echo -e "Detected CPU Cores: ${CYAN}${CPU_CORES}${NC}"
    echo -e "Total Physical RAM: ${CYAN}${TOTAL_RAM_GB} GB${NC}"
    echo -e "Pre-allocated Cluster RAM: ${CYAN}${PRE_ALLOCATED_RAM_GB} GB${NC}"
    echo -e "Effective Available RAM: ${CYAN}${EFFECTIVE_AVAILABLE_RAM_GB} GB${NC}"
    echo -e "Raw Free OS Disk: ${CYAN}${RAW_DISK_GB} GB${NC}"
    echo -e "Pre-allocated Cluster Disk: ${CYAN}${PRE_ALLOCATED_DISK_GB} GB${NC}"
    echo -e "Effective Available Disk: ${CYAN}${EFFECTIVE_DISK_GB} GB${NC}"

    # Auto-detect role uses effective available resources, but for backwards compat, use DISK_GB and RAM_GB mapping
    local RAM_GB=$TOTAL_RAM_GB
    local DISK_GB=$EFFECTIVE_DISK_GB

    # Auto-detect role if not explicitly set
    if [ -z "$ROLE" ]; then
        if [[ "$(hostname)" == *controller* ]]; then
            echo -e "${GREEN}✅ Hostname contains 'controller'. Defaulting role to 'all' and enabling full stack deployment.${NC}"
            ROLE="all"
            PROCESSED_ARGS+=("--role" "all" "--deploy-full-stack")
        elif [ "$RAM_GB" -le 4 ] || [ "$DISK_GB" -le 20 ]; then
            echo -e "${YELLOW}⚠️  Low resource machine detected ($RAM_GB GB RAM, $DISK_GB GB Disk). Defaulting role to 'worker', enabling external models and minimal stack.${NC}"
            ROLE="worker"
            PROCESSED_ARGS+=("--role" "worker" "--external-model-server" "--deploy-minimal-stack")
        elif [ "$RAM_GB" -ge 32 ] && [ "$CPU_CORES" -ge 4 ] && [ "$DISK_GB" -ge 500 ]; then
            echo -e "${GREEN}✅ Powerful machine detected. Defaulting role to 'all' and enabling full stack deployment.${NC}"
            ROLE="all"
            PROCESSED_ARGS+=("--role" "all" "--deploy-full-stack")
        else
            echo -e "${CYAN}ℹ️  Standard machine detected. Defaulting role to 'worker' and enabling partial stack deployment.${NC}"
            ROLE="worker"
            PROCESSED_ARGS+=("--role" "worker" "--deploy-partial-stack")
        fi
    else
        echo -e "Role explicitly set to: ${CYAN}${ROLE}${NC}"
    fi

    # Network Discovery & Role Fallback
    if [ "$ROLE" = "worker" ] && [ -z "$CONTROLLER_IP" ] && [ "$DO_STATUS" != true ]; then
        if find_controller; then
            # Controller found, CONTROLLER_IP is set. Add it to PROCESSED_ARGS.
            PROCESSED_ARGS+=("--controller-ip" "$CONTROLLER_IP")
        else
            echo -e "${YELLOW}⚠️  No controller found. Falling back to role 'all' (initializing as controller).${NC}"
            ROLE="all"

            # Remove the previous --role worker and add --role all
            local NEW_PROCESSED_ARGS=()
            local SKIP_NEXT=false
            for ((j=0; j<${#PROCESSED_ARGS[@]}; j++)); do
                local arg="${PROCESSED_ARGS[$j]}"
                if [ "$SKIP_NEXT" = true ]; then
                    SKIP_NEXT=false
                    continue
                fi
                if [ "$arg" = "--role" ]; then
                    NEW_PROCESSED_ARGS+=("--role" "all")
                    SKIP_NEXT=true
                elif [[ "$arg" =~ ^--role=.*$ ]]; then
                    NEW_PROCESSED_ARGS+=("--role" "all")
                else
                    NEW_PROCESSED_ARGS+=("$arg")
                fi
            done
            PROCESSED_ARGS=("${NEW_PROCESSED_ARGS[@]}")

            # Also add --deploy-full-stack since we are becoming the controller
            PROCESSED_ARGS+=("--deploy-full-stack")
        fi
    fi

    # Give some feedback about the network if it's set
    if [ -n "$CONTROLLER_IP" ]; then
         echo -e "Connecting to main controller at: ${BLUE}${CONTROLLER_IP}${NC}"
    fi
}

# --- Parse command-line arguments for wrapper logic ---
# We use a while loop to handle optional values for flags like --verbose
ARGS=("$@")
PROCESSED_ARGS=()
SKIP_NEXT=false

for ((i=0; i<${#ARGS[@]}; i++)); do
    arg="${ARGS[$i]}"

    if [ "$SKIP_NEXT" = true ]; then
        SKIP_NEXT=false
        continue
    fi

    case $arg in
        --status)
            DO_STATUS=true
            ;;
        --system-cleanup)
            DO_SYSTEM_CLEANUP=true
            DO_PURGE_JOBS=true
            DO_CLEAN_GIT=true
            ;;
        --deploy-full-stack)
            PROCESSED_ARGS+=("--deploy-full-stack")
            ;;
        --deploy-partial-stack)
            PROCESSED_ARGS+=("--deploy-partial-stack")
            ;;
        --deploy-minimal-stack)
            PROCESSED_ARGS+=("--deploy-minimal-stack")
            ;;
        --test-mode)
            PROCESSED_ARGS+=("--test-mode")
            ;;
        --dry-run)
            DO_DRY_RUN=true
            PROCESSED_ARGS+=("--dry-run")
            ;;
        --skip-setup)
            DO_SKIP_SETUP=true
            PROCESSED_ARGS+=("--skip-setup")
            ;;
        --tags)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                PROCESSED_ARGS+=("--tags" "$NEXT_ARG")
                ((i++))
            else
                echo -e "${RED}❌ Error: --tags requires an argument.${NC}"
                exit 1
            fi
            ;;
        --clean-git|--clean) # Support legacy --clean just in case, but map to clean-git
            DO_CLEAN_GIT=true
            # Don't pass to provisioning
            ;;
        --purge-jobs)
            DO_PURGE_JOBS=true
            # Don't pass to provisioning as a direct arg, we handle logic
            ;;
        --container)
            USE_CONTAINER=true
            PROCESSED_ARGS+=("$arg")
            ;;
        --debug)
            VERBOSE_LEVEL=4
            PROCESSED_ARGS+=("$arg")
            ;;
        --verbose)
            # Check next arg
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                VERBOSE_LEVEL="$NEXT_ARG"
                PROCESSED_ARGS+=("--verbose" "$VERBOSE_LEVEL")
                SKIP_NEXT=true
            else
                VERBOSE_LEVEL=3
                PROCESSED_ARGS+=("--verbose" "3")
            fi
            ;;
        --role)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                ROLE="$NEXT_ARG"
                PROCESSED_ARGS+=("--role" "$ROLE")
                SKIP_NEXT=true
            fi
            ;;
        --create-snapshot)
            DO_CREATE_SNAPSHOT=true
            ;;
        --list-snapshots)
            DO_LIST_SNAPSHOTS=true
            ;;
        --rollback-snapshot)
            DO_ROLLBACK_SNAPSHOT=true
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                ROLLBACK_SNAPSHOT_TARGET="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --github-ssh-user)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                PROCESSED_ARGS+=("--github-ssh-user" "$NEXT_ARG")
                SKIP_NEXT=true
            fi
            ;;
        --tier)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                TIER="$NEXT_ARG"
                PROCESSED_ARGS+=("--tier" "$TIER")
                SKIP_NEXT=true
            fi
            ;;
        --heal-cluster)
            DO_HEAL_CLUSTER=true
            ;;
        --troubleshoot)
            DO_TROUBLESHOOT=true
            for ((k=i+1; k<${#ARGS[@]}; k++)); do
                TROUBLESHOOT_ARGS+=("${ARGS[$k]}")
            done
            break
            ;;
        --recover-node)
            DO_RECOVER_NODE=true
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                RECOVER_NODE_IP="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --ipmi-host)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                IPMI_HOST="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --ipmi-user)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                IPMI_USER="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --ipmi-password)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                IPMI_PASSWORD="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --force-pxe)
            FORCE_PXE=true
            ;;
        --pxe-server-ip)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                PXE_SERVER_IP="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        -h|--help)
            show_help
            exit 0
            ;;
        -y|--yes)
            AUTO_YES=true
            ;;
        --agent-api-base)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                CLI_AGENT_API_BASE="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --agent-model)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                CLI_AGENT_MODEL="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        --agent-api-key)
            NEXT_ARG="${ARGS[$((i+1))]}"
            if [[ -n "$NEXT_ARG" && ! "$NEXT_ARG" =~ ^- ]]; then
                CLI_AGENT_API_KEY="$NEXT_ARG"
                SKIP_NEXT=true
            fi
            ;;
        *)
            PROCESSED_ARGS+=("$arg")
            ;;
    esac
done

# Apply CLI arguments to environment variables if provided
if [ -n "$CLI_AGENT_API_BASE" ]; then
    export AGENT_API_BASE="$CLI_AGENT_API_BASE"
fi
if [ -n "$CLI_AGENT_MODEL" ]; then
    export AGENT_MODEL="$CLI_AGENT_MODEL"
fi
if [ -n "$CLI_AGENT_API_KEY" ]; then
    export AGENT_API_KEY="$CLI_AGENT_API_KEY"
fi

# --- Move to the script's directory ---
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd "$SCRIPT_DIR" || exit 1

# Run system profiling before we proceed, unless just asking for status

profile_system


LOG_FILE="bootstrap_debug.log"
AGENT_LOG="agent_recovery.log"
true > "$LOG_FILE"

# --- Error Handling & Auto-Recovery ---
# shellcheck disable=SC2317
handle_error() {
    local error_line="$1"

    # Avoid recursive error loops
    if [ "${IN_ERROR_HANDLER:-0}" -eq 1 ]; then
        echo -e "\n${RED}❌ Critical error inside the recovery handler. Halting to prevent infinite loop.${NC}"
        exit 1
    fi
    # Use local variable instead of export to prevent state leakage across exec
    local IN_ERROR_HANDLER=1

    echo -e "\n${BOLD}${RED}⚠️  Bootstrap failed at line ${error_line}!${NC}"
    echo -e "${YELLOW}Initiating autonomous recovery via smol_agent...${NC}"

    # Extract log context
    local log_snippet
    log_snippet=$(tail -n 50 "$LOG_FILE" 2>/dev/null || echo "No log output available.")

    local prompt="You are an autonomous debugging agent. The bootstrap.sh script just failed at line ${error_line}.
Here is the recent log output context:

\`\`\`
${log_snippet}
\`\`\`

Your task is to:
1. Analyze the error carefully.
2. Fix the underlying issue by modifying bootstrap.sh, Ansible playbooks, or other configuration files as necessary.
3. Commit the changes to the Git repository with a descriptive commit message, and explicitly push them to the remote repository.
4. Do not ask for user confirmation, execute the fix autonomously and exit.
"

    echo -e "Agent thought process and actions will be logged to: ${CYAN}${AGENT_LOG}${NC}"
    echo -e "⏳ Please wait while the agent attempts to fix the issue..."

    local agent_status=0
    # Run SmolAgentTool headlessly via the wrapper script.
    python3 "$SCRIPT_DIR/scripts/run_smol_recovery.py" "$prompt" > "$AGENT_LOG" 2>&1 || agent_status=$?

    if [ "$agent_status" -eq 0 ]; then
        echo -e "${GREEN}✅ Autonomous recovery successful!${NC}"
        echo -e "${CYAN}Resuming bootstrap process...${NC}"
        # Prevent trap from firing again during exec
        trap - ERR
        # Re-execute the script, appending --continue. Fallback to empty if ARGS is not defined yet.
        exec "$0" "${ARGS[@]:-}" "--continue"
    else
        echo -e "${RED}❌ Autonomous recovery failed. Please review ${AGENT_LOG} for details.${NC}"
        exit 1
    fi
}

# Set the global trap
trap 'handle_error $LINENO' ERR

# --- Helper: Run Step ---
run_step() {
    local desc="$1"
    local cmd="$2"

    # Levels 3 and 4 show output
    if [ "$VERBOSE_LEVEL" -ge 3 ]; then
        echo -e "\n${BOLD}${CYAN}--- Running ${desc} ---${NC}"
        # Execute and tee to log
        eval "$cmd" 2>&1 | tee -a "$LOG_FILE"
        local status=${PIPESTATUS[0]} # Capture exit code of the evaluated command

        if [ "$status" -eq 0 ]; then
            echo -e "${GREEN}✅ ${desc} complete.${NC}"
        else
            echo -e "${RED}❌ ${desc} failed.${NC}"
            return "$status"
        fi
    else
        echo -n -e "⏳ ${desc}..."

        local tmp_log
        tmp_log=$(mktemp)
        eval "$cmd" > "$tmp_log" 2>&1
        local status=$?

        cat "$tmp_log" >> "$LOG_FILE"

        if [ $status -eq 0 ]; then
            echo -e "\r\033[K${GREEN}✅ ${desc} Complete${NC}"
        else
            echo -e "\r\033[K${RED}❌ ${desc} Failed${NC}"
            echo -e "${YELLOW}--- Error Log ---${NC}"
            cat "$tmp_log"
            echo -e "${YELLOW}-----------------${NC}"
            rm "$tmp_log"
            return $status
        fi
        rm "$tmp_log"
    fi
}

ask_confirm() {
    local prompt="$1"
    if [ "$AUTO_YES" = true ]; then
        echo -e "${prompt} [y/N] ${GREEN}y (auto-yes)${NC}"
        return 0
    fi
    read -p "$prompt [y/N] " -n 1 -r
    echo
    if [[ $REPLY =~ ^[Yy]$ ]]; then
        return 0
    else
        return 1
    fi
}

# --- Git Hooks Setup ---
setup_git_hooks() {
    if [ -d ".git" ]; then
        git config core.hooksPath .githooks
        echo -e "${GREEN}✅ Configured git to use .githooks for hooks.${NC}"
    fi
}

# --- Environment Setup (Reusable) ---
VENV_DIR="$SCRIPT_DIR/.venv"

# shellcheck disable=SC2317
setup_venv() {
    if [ ! -d "$VENV_DIR" ]; then
        # If a virtual environment is active but the directory is gone, deactivate it
        if [ -n "$VIRTUAL_ENV" ]; then
            if type deactivate >/dev/null 2>&1; then
                deactivate
            fi
            # Clear hashed executable paths (like python3)
            hash -r 2>/dev/null || true
        fi

        if command -v python3.12 >/dev/null 2>&1; then
            python3.12 -m venv "$VENV_DIR"
        else
            python3 -m venv "$VENV_DIR"
        fi
    fi
}

ensure_python_environment() {
    echo -e "\n${BOLD}=== Environment Setup ===${NC}"
    chmod o-w . ansible.cfg

    run_step "Setting up Git Hooks" "setup_git_hooks"
    run_step "Creating Python virtual environment" "setup_venv"

    # Activate venv for this script execution
    # shellcheck disable=SC1091
    source "$VENV_DIR/bin/activate"

    run_step "Upgrading pip" "pip install --upgrade pip"

    if [ -f "requirements-dev.txt" ]; then
        run_step "Installing Python dependencies" "pip install -r requirements-dev.txt --verbose"
    else
        echo "⚠️  Warning: requirements-dev.txt not found. Skipping dependency installation."
    fi

    if [ "$DO_DRY_RUN" != true ]; then
        run_step "Installing Node.js Environment" "pip install nodeenv && (command -v node >/dev/null 2>&1 || nodeenv -p)"
        run_step "Installing OpenCode AI Agent" "npm install"
    fi

    run_step "Installing Ansible Core" "pip install ansible-core pyyaml resolvelib"

    # --- Find Ansible Playbook executable ---
    ANSIBLE_GALAXY_EXEC="$VENV_DIR/bin/ansible-galaxy"

    # Install Ansible collections
    if [ -x "$ANSIBLE_GALAXY_EXEC" ]; then
        export ANSIBLE_CONFIG="$(pwd)/ansible.cfg"
        run_step "Installing Ansible collections" "$ANSIBLE_GALAXY_EXEC collection install community.general ansible.posix community.docker community.sops community.crypto"
    else
        echo "Error: ansible-galaxy not found at $ANSIBLE_GALAXY_EXEC." >&2
        exit 1
    fi
}

# --- Cleanup Actions ---

perform_purge_jobs() {
    echo -e "\n${BOLD}${YELLOW}⚠️  Purge Jobs initiated.${NC}"
    if ask_confirm "Are you sure you want to stop and purge all Nomad jobs?"; then
        # Ensure we have the python environment to run the script
        ensure_python_environment

        echo "Running provisioning script to purge jobs..."
        # Pass --purge-jobs and --only-purge
        python3 scripts/provisioning.py --purge-jobs --only-purge
        local status=$?
        if [ $status -eq 0 ]; then
            echo -e "${GREEN}✅ Jobs purged.${NC}"
        else
             echo -e "${RED}❌ Job purge failed.${NC}"
        fi
    else
        echo "Job purge cancelled."
    fi
}

perform_system_cleanup() {
    echo -e "\n${BOLD}${YELLOW}⚠️  System Cleanup initiated (Docker, Apt, Logs).${NC}"
    if ask_confirm "Are you sure you want to aggressively clean system resources?"; then
        if [ -x "scripts/cleanup.sh" ]; then
            # We assume the user has sudo if they are running this
            run_step "Running system cleanup script" "sudo ./scripts/cleanup.sh"
        else
            echo -e "${RED}❌ scripts/cleanup.sh not found or not executable.${NC}"
        fi
    else
        echo "System cleanup cancelled."
    fi
}

perform_git_clean() {
    echo -e "\n${BOLD}${YELLOW}⚠️  Git Clean initiated.${NC}"
    echo "This will permanently delete all untracked files."
    echo "--------------------------------------------------"
    git clean -ndx
    echo "--------------------------------------------------"

    if ask_confirm "Are you sure you want to permanently delete these files?"; then
        if ! run_step "Cleaning repository" "git clean -fdx"; then
            echo -e "\n${YELLOW}⚠️  Standard cleanup failed. This is often due to files created with sudo.${NC}"
            if ask_confirm "Do you want to try cleaning with sudo?"; then
                # Ensure sudo credentials
                if ! sudo -n true 2>/dev/null; then
                    sudo -v
                fi
                run_step "Cleaning repository (with sudo)" "sudo git clean -fdx"
            else
                echo "Cleanup skipped."
            fi
        fi
    else
        echo "Git clean cancelled."
    fi
}

# --- Status Action ---
if [ "$DO_STATUS" = true ]; then
    echo -e "${BOLD}=== Cluster Status ===${NC}"
    ensure_python_environment
    python3 scripts/provisioning.py --only-status "${PROCESSED_ARGS[@]}"

    echo -e "\n${BOLD}=== Opencode Status ===${NC}"
    python3 scripts/troubleshoot.py opencode-status
    exit 0
fi

# --- OS Recovery Snapshot Actions ---
if [ "$DO_CREATE_SNAPSHOT" = true ]; then
    echo -e "${BOLD}=== Creating Pre-deployment Snapshot ===${NC}"
    sudo python3 scripts/recover_os.py --create
    exit $?
fi

if [ "$DO_LIST_SNAPSHOTS" = true ]; then
    echo -e "${BOLD}=== Listing Snapshots ===${NC}"
    sudo python3 scripts/recover_os.py --list
    exit $?
fi

if [ "$DO_ROLLBACK_SNAPSHOT" = true ]; then
    echo -e "${BOLD}=== Rolling back OS Snapshot ===${NC}"
    if [ -n "$ROLLBACK_SNAPSHOT_TARGET" ]; then
        sudo python3 scripts/recover_os.py --rollback "$ROLLBACK_SNAPSHOT_TARGET"
    else
        sudo python3 scripts/recover_os.py --rollback latest
    fi
    exit $?
fi

# --- Recovery Actions ---
if [ "$DO_HEAL_CLUSTER" = true ]; then
    echo -e "${BOLD}=== Cluster Healing ===${NC}"
    ./scripts/heal_cluster.sh
    exit $?
fi

if [ "$DO_TROUBLESHOOT" = true ]; then
    echo -e "${BOLD}=== Cluster Troubleshooting & Healing Utility ===${NC}"
    ensure_python_environment
    python3 scripts/troubleshoot.py "${TROUBLESHOOT_ARGS[@]}"
    exit $?
fi

if [ "$DO_RECOVER_NODE" = true ]; then
    echo -e "${BOLD}=== Node Recovery ===${NC}"
    ensure_python_environment

    RECOVER_CMD=(python3 scripts/recover_node.py --node-ip "$RECOVER_NODE_IP")
    if [ -n "$IPMI_HOST" ]; then
        RECOVER_CMD+=(--ipmi-host "$IPMI_HOST")
    fi
    if [ -n "$IPMI_USER" ]; then
        RECOVER_CMD+=(--ipmi-user "$IPMI_USER")
    fi
    if [ -n "$IPMI_PASSWORD" ]; then
        RECOVER_CMD+=(--ipmi-password "$IPMI_PASSWORD")
    fi
    if [ "$FORCE_PXE" = true ]; then
        RECOVER_CMD+=(--force-pxe)
    fi
    if [ -n "$PXE_SERVER_IP" ]; then
        RECOVER_CMD+=(--pxe-server-ip "$PXE_SERVER_IP")
    fi

    "${RECOVER_CMD[@]}"
    exit $?
fi

# --- Execute Cleanup Actions ---
# We execute these BEFORE everything else to ensure a clean slate if requested.

if [ "$DO_PURGE_JOBS" = true ]; then
    perform_purge_jobs
fi

if [ "$DO_SYSTEM_CLEANUP" = true ]; then
    perform_system_cleanup
fi

if [ "$DO_CLEAN_GIT" = true ]; then
    perform_git_clean
fi

# If the user only requested cleanup, we might want to stop here?
# But typically bootstrap means "setup". If I wanted to JUST clean, I might not expect it to start building again.
# However, for now we follow the pattern: cleanup then proceed.


# --- Container Mode ---
if [ "$USE_CONTAINER" = true ]; then
    echo -e "${BOLD}--- Running in Container Mode ---${NC}"

    # --- Host-side Cluster Detection ---
    check_for_existing_cluster() {
        echo "--- Checking for existing cluster on host ---"
        CLUSTER_EXISTS=false
        if nc -z localhost 4646 2>/dev/null || nc -z localhost 8500 2>/dev/null; then
            echo "✅ Detected existing Nomad/Consul service on the host."
            CLUSTER_EXISTS=true
        else
            echo "ℹ️  No existing cluster detected on the host. Will start a new one."
        fi
    }

    # Check if we are already inside the container
    if [ -f "/.dockerenv" ] && [ "$(hostname)" = "pipecat-dev-runner" ]; then
        echo "✅ Already inside the container. Proceeding with bootstrap..."
    else
        IMAGE_NAME="pipecat-dev-container"
        CONTAINER_NAME="pipecat-dev-runner"

        check_for_existing_cluster

        echo "Building container image: $IMAGE_NAME..."
        if ! docker build -t "$IMAGE_NAME" docker/dev_container/; then
            echo "❌ Failed to build container image."
            exit 1
        fi

        echo "Checking for existing container..."
        if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
            echo "Removing existing container..."
            docker rm -f "$CONTAINER_NAME"
        fi

        # --- Dynamically configure and start container ---
        HOST_IP=$(hostname -I | awk '{print $1}')
        if [ -z "$HOST_IP" ]; then
            echo "❌ Error: Could not determine host IP address."
            exit 1
        fi

        DOCKER_RUN_CMD=(docker run -d --rm --privileged --name "$CONTAINER_NAME" \
            --hostname "$CONTAINER_NAME" \
            -v /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host \
            -v "$SCRIPT_DIR":/opt/cluster-infra -e "HOST_IP=$HOST_IP")

        if [ "$CLUSTER_EXISTS" = true ]; then
            echo "Configuring container as a WORKER to join the existing cluster."
        else
            echo "Configuring container as a new CONTROLLER."
            DOCKER_RUN_CMD+=(-p 4646:4646 -p 8500:8500 -p 8081:8081 -p 8000:8000)
        fi

        DOCKER_RUN_CMD+=("$IMAGE_NAME")

        echo "Starting container..."
        if ! "${DOCKER_RUN_CMD[@]}"; then
            echo "❌ Failed to start container."
            exit 1
        fi

        echo "Waiting for container to initialize..."
        sleep 5

        echo "Executing bootstrap inside the container..."
        # Pass processed args
        docker exec -it "$CONTAINER_NAME" /bin/bash -c "cd /opt/cluster-infra && ./bootstrap.sh ${PROCESSED_ARGS[*]}"
        EXIT_CODE=$?

        echo "Container bootstrap finished with exit code: $EXIT_CODE"
        echo "You can access the container using: docker exec -it $CONTAINER_NAME /bin/bash"
        echo "To stop and remove the container: docker rm -f $CONTAINER_NAME"

        exit $EXIT_CODE
    fi
fi

# --- Run Initial Machine Setup ---
echo -e "${BOLD}=== System Bootstrap ===${NC}"
if [ "$DO_SKIP_SETUP" = true ]; then
    echo -e "⏭️  Skipping initial machine setup script due to --skip-setup flag."
elif [ -f "initial-setup/setup.sh" ]; then
    if [ "$DO_DRY_RUN" = true ]; then
        echo -e "⏭️  Skipping initial machine setup script due to --dry-run."
    elif [ -f "/.dockerenv" ] && [ "$(hostname)" = "pipecat-dev-runner" ]; then
         echo "🐳 Container environment detected. Skipping initial machine setup (setup.sh)."
    else
        # We need to ensure sudo doesn't hang on prompt hidden by redirection
        if sudo -n true 2>/dev/null; then
            # Sudo is already cached
            run_step "Initial machine setup" "sudo bash initial-setup/setup.sh"
        else
            echo "You may be prompted for your sudo password to run the initial setup script."
            sudo -v
            run_step "Initial machine setup" "sudo bash initial-setup/setup.sh"
        fi
    fi
else
    echo "⚠️  Warning: initial-setup/setup.sh not found. Skipping pre-configuration."
fi

# --- Install Python dependencies (Virtual Environment) ---
# Ensure environment is ready (it might have been set up by purge_jobs, or deleted by git clean)
ensure_python_environment

# --- Run Provisioning Script ---
# echo "🚀 Handing over to Python provisioning script..."
# Pass processed arguments to the python script.
python3 scripts/provisioning.py "${PROCESSED_ARGS[@]}"
EXIT_CODE=$?

if [ $EXIT_CODE -eq 0 ]; then
    echo -e "\n${GREEN}✨ Bootstrap complete.${NC}"

    # Mark the system as bootstrapped to prevent re-running the systemd service
    sudo touch /var/lib/pipecat_bootstrapped

    # Run Post-Bootstrap Cleanup
    if [ -x "scripts/cleanup.sh" ]; then
        echo -e "\n${BOLD}${CYAN}🧹 Initiating Post-Bootstrap Space Reclamation...${NC}"
        # Execute non-destructive cache and temp file sweep
        if sudo -n true 2>/dev/null; then
            sudo ./scripts/cleanup.sh --post-bootstrap
        else
            echo "You may be prompted for your sudo password to run post-bootstrap cleanup."
            sudo ./scripts/cleanup.sh --post-bootstrap
        fi
    fi
else
    echo -e "\n${RED}❌ Bootstrap failed with exit code $EXIT_CODE.${NC}"
fi

exit $EXIT_CODE