bot

#!/bin/bash
# bot — start/stop/status for the PyCheckmate Lichess bot.
#
# Why this exists instead of launchd: lichess-bot spawns multiprocessing
# children (the event-stream watcher among them). If the main process dies
# without cleaning up, those children survive as orphans and keep hitting
# /api/stream/event forever, which renews Lichess's 429 penalty no matter
# how long you "wait". Every up/down here therefore sweeps the whole
# process tree, and nothing auto-restarts behind your back.
set -u

# The two directories this script drives are named outright rather than derived
# from where the script itself sits. It used to resolve its own symlink to find
# the lichess-bot clone, which worked only while the canonical copy lived there;
# once this repo became the single versioned source, that lookup pointed at
# PyCheckmate and `bot up` died on a missing run.sh.
#
# They stay explicit, but are overridable so a fresh checkout on a differently
# organised machine needs no edit to this file — the same two variables
# deploy/setup.sh accepts.
BOT_DIR="${LICHESS_BOT_DIR:-$HOME/PycharmProjects/lichess-bot}"   # the clone: run.sh, .venv, .token
PIDFILE="$BOT_DIR/bot.pid"
LOG="$BOT_DIR/bot.log"
LOG_MAX_BYTES=52428800   # 50 MB; see the rotation note in `bot up`
# The Stockfish analysis watcher runs alongside the bot: it grades each game
# in the idle gap after it finishes, so a multi-day unattended run arrives
# with its analysis already done. It lives in the PyCheckmate repo's venv, not
# lichess-bot's, so the venv sweep below cannot see it -- hence its own pid.
ENGINE_DIR="${PYCHECKMATE_DIR:-$HOME/PycharmProjects/PyCheckmate}"
# The engine process `bot down` waits on before stopping, so a game in progress
# is never killed and forfeited. PyCheckmate runs as `python -m engine.uci`;
# CPyCheckmate is a native binary, so its `cbot` wrapper overrides this with the
# deployed path. Leave it matching only the long-running engine -- too broad a
# pattern makes `bot down` wait forever on something that is not a game.
ENGINE_PROC="${BOT_ENGINE_PROC:-engine.uci}"
WATCHPID="$BOT_DIR/sf_watch.pid"
WATCHLOG="$HOME/.local/share/pycheckmate/sf_watch.log"
MATCH="$BOT_DIR/.venv"   # every bot process (main + children) runs this venv's python

# Pin the label each game is recorded under, e.g. BOT_ENGINE_VERSION=fd9ebc0.
#
# sf_watch normally stamps records with the current git rev, and starting a new
# rev starts a new measurement set. That is right when the engine changed, and
# wrong when it did not: committing docs or anything under engine/tools/ during
# a run moves HEAD without altering a single line the engine executes, so a
# restart would split one set of games into two labels that cannot be averaged
# ("never average across engine_version"). Pinning keeps them one set.
#
# Only pin when `git diff <pinned-rev> HEAD -- engine/` shows nothing outside
# engine/tools/. If the engine itself changed, the new label is correct and
# this must be unset -- a stale pin would silently file new games under the old
# engine, which is the exact error the label exists to prevent.
#
# `bot up` writes the pin to VERSIONFILE and `bot down` reads it back, because
# BOT_ENGINE_VERSION is a per-invocation environment variable and is long gone
# by the time you stop the bot days later. Without that the drain query falls
# back to the newest cut rather than the pinned one, so any game finishing
# between the two timestamps is invisible to the pending count -- `bot down`
# then reports nothing to wait for and kills the watcher mid-analysis, losing
# that game's record entirely.
ENGINE_VERSION="${BOT_ENGINE_VERSION:-}"
VERSIONFILE="$BOT_DIR/engine_version"

alive() { [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; }
# The watcher is launched through `uv run`, which forks, so the pid the
# shell sees is a wrapper rather than the python process. Matching on the
# module path finds the real one and needs no pidfile to go stale.
# Match the long-running watcher only. The shutdown path spawns a short
# `-m engine.tools.sf_watch --pending` query, which a bare pattern would also
# match -- making the watcher look alive because we just asked it a
# question, and leaving stop_watch spinning against its own probe.
watch_procs() { pgrep -f "engine.tools.sf_watch --since" 2>/dev/null; }
watch_alive() { [ -n "$(watch_procs)" ]; }

stop_watch() {
    pkill -f "engine.tools.sf_watch" 2>/dev/null
    for _ in 1 2 3 4 5; do
        [ -z "$(watch_procs)" ] && break
        sleep 1
    done
    pkill -9 -f "engine.tools.sf_watch" 2>/dev/null
    rm -f "$WATCHPID"
}
procs() { pgrep -f "$MATCH" 2>/dev/null; }

sweep() {
    # Kill the pidfile process, then every venv process (catches orphans).
    [ -f "$PIDFILE" ] && kill "$(cat "$PIDFILE")" 2>/dev/null
    pkill -f "$MATCH" 2>/dev/null
    for _ in 1 2 3 4 5; do
        [ -z "$(procs)" ] && break
        sleep 1
    done
    pkill -9 -f "$MATCH" 2>/dev/null
    rm -f "$PIDFILE"
}

case "${1:-}" in
up)
    if alive; then
        echo "already running (pid $(cat "$PIDFILE")) — use 'bot down' first"
        exit 1
    fi
    # Never two bots at once, even on different Lichess accounts. Two engines
    # thinking simultaneously contend for this machine's four performance cores
    # and distort each other's clocks — the same effect that made SPRT
    # concurrency 4 forfeit games on time. Rated games measured under that
    # contention are worthless, and PyCheckmate's are the ones its README and
    # the LinkedIn write-up quote.
    for other in "$HOME"/PycharmProjects/lichess-bot*/bot.pid; do
        [ "$other" = "$PIDFILE" ] && continue
        [ -f "$other" ] || continue
        if kill -0 "$(cat "$other")" 2>/dev/null; then
            echo "another bot is running: $other (pid $(cat "$other"))"
            echo "stop it first — two engines at once corrupt both sides' clocks"
            exit 1
        fi
    done
    strays="$(procs)"
    if [ -n "$strays" ]; then
        echo "killing stray bot processes: $(echo "$strays" | tr '\n' ' ')"
        sweep
        echo "waiting 60s so Lichess's rate-limit window clears before reconnecting..."
        sleep 60
    fi
    cd "$BOT_DIR" || exit 1
    # Rotate before starting, keeping one generation. run.sh passes -v, so
    # lichess-bot logs at DEBUG and echoes every UCI `info` line the engine
    # emits -- one per deepening iteration, per move, per game. That is worth
    # having (those lines exist because two bugs stayed invisible for weeks
    # without them), but `bot up` appends and nothing else ever truncates, so
    # the log had reached 129 MB / 992k lines before anyone looked.
    if [ -f "$LOG" ] && [ "$(stat -f%z "$LOG")" -gt "$LOG_MAX_BYTES" ]; then
        mv -f "$LOG" "$LOG.1"
        echo "rotated $(basename "$LOG") -> $(basename "$LOG").1"
    fi
    # run.sh loads the token from .token and execs the venv python;
    # caffeinate -s keeps the Mac awake (on AC power) while the bot runs.
    nohup caffeinate -s "$BOT_DIR/run.sh" >> "$LOG" 2>&1 &
    echo $! > "$PIDFILE"
    # Only grade games this engine build actually played. --since auto anchors
    # the cut to when this build was first deployed, not to process start, so
    # restarting never orphans a game that finished just before a shutdown --
    # while a new build still begins a clean set. Each record is stamped with
    # the build, so one file can never average two engines.
    mkdir -p "$(dirname "$WATCHLOG")"
    if [ -n "$ENGINE_VERSION" ]; then
        echo "$ENGINE_VERSION" > "$VERSIONFILE"
    else
        rm -f "$VERSIONFILE"
    fi
    ( cd "$ENGINE_DIR" && PYTHONPATH=. nohup uv run --no-project python -u \
        -m engine.tools.sf_watch --since auto \
        ${ENGINE_VERSION:+--version "$ENGINE_VERSION"} >> "$WATCHLOG" 2>&1 & )
    [ -n "$ENGINE_VERSION" ] && echo "recording games as engine version $ENGINE_VERSION"
    sleep 3
    echo "started (pid $(cat "$PIDFILE")), keeping Mac awake"
    if watch_alive; then
        echo "analysis watcher ($(watch_procs | tr '\n' ' ')) grading games as they finish"
    else
        echo "WARNING: analysis watcher did not start -- see $WATCHLOG"
    fi
    echo "'bot log' to watch, 'bot down' to stop both"
    ;;
down)
    force=0
    [ "${2:-}" = "now" ] && force=1

    # Graceful by default: killing mid-game forfeits it, and killing the
    # watcher mid-analysis loses that game's record entirely (a record is only
    # written once the whole game is graded). So wait for both to reach a
    # clean point, and say so on screen -- a silent wait is indistinguishable
    # from a hang.
    if [ "$force" -eq 0 ] && alive; then
        waited=0
        while pgrep -f "$ENGINE_PROC" >/dev/null 2>&1; do
            printf "\rwaiting for the current game to finish... %dm%02ds  (bot down now = don't wait)" \
                $((waited / 60)) $((waited % 60))
            sleep 5
            waited=$((waited + 5))
        done
        # \033[K clears from the cursor to end of line. Padding with spaces is
        # what this used to do, and it left the tail of the longer progress
        # line on screen -- the countdown is 75 columns and the pad covered 30.
        [ "$waited" -gt 0 ] && printf "\rgame finished after %dm%02ds\033[K\n" \
            $((waited / 60)) $((waited % 60))
    fi

    # Stop the bot first so no new game can start while the analysis drains.
    sweep_done=0
    if alive || [ -n "$(procs)" ]; then
        sweep
        sweep_done=1
    fi

    if [ "$force" -eq 0 ] && watch_alive; then
        # Same pin the watcher was started with, so both halves resolve the
        # same cut. See the VERSIONFILE note at the top.
        [ -f "$VERSIONFILE" ] && ENGINE_VERSION="$(cat "$VERSIONFILE")"
        waited=0
        while :; do
            left=$(cd "$ENGINE_DIR" && PYTHONPATH=. uv run --no-project python \
                     -m engine.tools.sf_watch --pending --since auto \
                     ${ENGINE_VERSION:+--version "$ENGINE_VERSION"} 2>/dev/null | tail -1)
            case "$left" in ''|*[!0-9]*) left=0 ;; esac
            # Wait on the pending count alone. Checking for a live stockfish
            # process looks safer but hangs forever: sf_watch keeps one client
            # alive between games on purpose, so it is idle-but-present most of
            # the time. A record is only written once a whole game is graded, so
            # pending==0 already means nothing is mid-analysis.
            [ "$left" -eq 0 ] && break
            printf "\rwaiting for Stockfish analysis... %d game(s) left  %dm%02ds" \
                "$left" $((waited / 60)) $((waited % 60))
            sleep 10
            waited=$((waited + 10))
        done
        [ "$waited" -gt 0 ] && printf "\ranalysis complete after %dm%02ds\033[K\n" \
            $((waited / 60)) $((waited % 60))
    fi
    stop_watch

    if [ "$sweep_done" -eq 1 ]; then
        left="$(procs)"
        if [ -z "$left" ]; then
            echo "stopped, all children killed"
        else
            echo "WARNING: processes still alive: $left"
            exit 1
        fi
        exit 0
    fi

    if ! alive && [ -z "$(procs)" ]; then
        echo "not running"
        rm -f "$PIDFILE"
        exit 0
    fi
    ;;
status)
    if watch_alive; then
        echo "analysis watcher: running ($(watch_procs | tr '\n' ' '))"
    else
        echo "analysis watcher: stopped"
    fi
    if alive; then
        echo "running (pid $(cat "$PIDFILE"), $(procs | wc -l | tr -d ' ') processes)"
    elif [ -n "$(procs)" ]; then
        echo "STRAY processes without pidfile (run 'bot down'): $(procs | tr '\n' ' ')"
    else
        echo "stopped"
    fi
    if [ -f "$LOG" ]; then
        rl=$(tail -200 "$LOG" | grep -c "rate-limit\|429" || true)
        echo "recent rate-limit lines (last 200 log lines): $rl"
        echo "--- last log lines ---"
        tail -5 "$LOG"
    fi
    ;;
log)
    tail -f "$LOG"
    ;;
*)
    echo "usage: bot up | bot down [now] | bot status | bot log"
    exit 1
    ;;
esac