load-online-migrate.yml

name: load-online-migrate
on:
  push:
    branches:
      - main
    paths:
      - "cmd/hatchet-migrate/migrate/migrations/**"
      - "cmd/hatchet-loadtest/**"
      - "hack/ci/load-online-migrate.sh"
      - ".github/workflows/load-online-migrate.yml"
  pull_request:
    paths:
      - "cmd/hatchet-migrate/migrate/migrations/**"
      - "cmd/hatchet-loadtest/**"
      - "hack/ci/load-online-migrate.sh"
      - ".github/workflows/load-online-migrate.yml"

# Models the deploy order: migrations always run before new engines roll out.
# The BASE commit's engine (the PR base, or the previous main commit on push)
# runs on the base schema under load, this branch's migrations are applied
# online against it, and only then does this branch's engine take over on the
# migrated schema. See hack/ci/load-online-migrate.sh for the phases.
jobs:
  load-online-migrate:
    runs-on: ubicloud-standard-8
    timeout-minutes: 40
    permissions:
      contents: read
      pull-requests: write
      issues: write
    env:
      DATABASE_URL: postgresql://hatchet:hatchet@127.0.0.1:5431/hatchet?sslmode=disable
      SERVER_GRPC_PORT: "7077"
      SERVER_GRPC_BROADCAST_ADDRESS: localhost:7077
      SERVER_SECURITY_CHECK_ENABLED: "false"
      SERVER_GRPC_INSECURE: "true"
      SERVER_AUTH_COOKIE_DOMAIN: localhost
      SERVER_AUTH_COOKIE_INSECURE: "true"
      SERVER_MSGQUEUE_KIND: postgres
      SERVER_LOGGER_LEVEL: warn
      SERVER_LOGGER_FORMAT: console
      DATABASE_LOGGER_LEVEL: warn
      DATABASE_LOGGER_FORMAT: console
      BASE_DIR: /tmp/hatchet-base
      BASE_BIN: /tmp/hatchet-bin/base
      PR_BIN: /tmp/hatchet-bin/pr

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # push: HEAD^1 is the previous main commit, the engine that is
          # running when this commit's migrations are applied.
          fetch-depth: 2

      # The base commit is what production runs when this change's migrations
      # are applied: the PR base for pull requests, the parent commit on main.
      - name: Resolve base commit
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BASE_SHA="${{ github.event.pull_request.base.sha }}"
            git fetch --no-tags --depth=1 origin "$BASE_SHA"
          else
            BASE_SHA=$(git rev-parse HEAD^1)
          fi
          echo "Base commit: $BASE_SHA"
          echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV

          git worktree add "$BASE_DIR" "$BASE_SHA"

          BASE_MIGRATION_VERSION=$(ls "$BASE_DIR/cmd/hatchet-migrate/migrate/migrations/" \
            | grep -E '^[0-9]+' \
            | sort \
            | tail -1 \
            | cut -d'_' -f1)
          if [ -z "$BASE_MIGRATION_VERSION" ]; then
            echo "could not determine latest migration on $BASE_SHA"
            exit 1
          fi
          echo "Base migration version ($BASE_SHA): $BASE_MIGRATION_VERSION"
          echo "BASE_MIGRATION_VERSION=$BASE_MIGRATION_VERSION" >> $GITHUB_ENV

      - name: Setup Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version-file: go.mod

      - name: Compose
        run: docker compose up -d

      # The base migrator and admin set the database up the way production had
      # it before this change; the base engine is what runs while the branch
      # migrations are applied.
      - name: Build base binaries
        working-directory: ${{ env.BASE_DIR }}
        run: |
          mkdir -p "$BASE_BIN"
          go build -o "$BASE_BIN/" ./cmd/hatchet-engine ./cmd/hatchet-migrate ./cmd/hatchet-admin

      - name: Build branch binaries
        run: |
          mkdir -p "$PR_BIN"
          go build -o "$PR_BIN/" ./cmd/hatchet-engine ./cmd/hatchet-migrate ./cmd/hatchet-loadtest

      - name: Run base migrations
        run: ${{ env.BASE_BIN }}/hatchet-migrate

      - name: Setup config and seed database
        run: |
          mkdir -p generated
          ${{ env.BASE_BIN }}/hatchet-admin quickstart --skip certs --generated-config-dir ./generated

      - name: Generate API token
        run: |
          TOKEN=$(${{ env.BASE_BIN }}/hatchet-admin --config ./generated token create)
          echo "HATCHET_CLIENT_TOKEN=$TOKEN" >> $GITHUB_ENV

      - name: Run load test with migrations
        id: run-load-test
        continue-on-error: true
        run: bash ./hack/ci/load-online-migrate.sh
        env:
          HATCHET_CLIENT_TOKEN: ${{ env.HATCHET_CLIENT_TOKEN }}
          LOG_DIR: /tmp

      - name: Upload load-online-migrate logs
        if: steps.run-load-test.outcome == 'failure'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: load-online-migrate-logs
          path: |
            /tmp/engine-base.log
            /tmp/engine-pr.log
            /tmp/loadtest-base.log
            /tmp/loadtest-pr.log
          if-no-files-found: ignore

      - name: Comment on PR
        if: steps.run-load-test.outcome == 'failure' && github.event_name == 'pull_request'
        run: |
          gh pr comment ${{ github.event.pull_request.number }} \
            --body "⚠️ **Optional test failure**: The \`load-online-migrate\` job failed on this PR. This check is non-mandatory and does not block merging, but may be worth investigating. [View logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})"
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Teardown
        if: always()
        run: |
          if [ -f /tmp/loadtest.pid ]; then kill $(cat /tmp/loadtest.pid) 2>/dev/null || true; fi
          if [ -f /tmp/engine.pid ]; then kill $(cat /tmp/engine.pid) 2>/dev/null || true; fi
          docker compose down