Repository navigation
perf: scale agent execution with typed execution state, a claim dispatcher, and owned continuation #559
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Live Tests | |
| # Runs provider-backed live tests against real credentials. They run on every | |
| # push to main and on manual dispatch. On pull requests they are opt-in: add the | |
| # `live-tests` label to the PR. Adding an unrelated label to an already-labeled | |
| # PR does not retrigger a run. Only users with triage access or higher can add | |
| # labels. Pull requests from forks never receive repository secrets, so the | |
| # jobs fail early with a clear message instead of silently skipping. | |
| on: | |
| pull_request: | |
| types: | |
| - labeled | |
| - synchronize | |
| - reopened | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| live-web: | |
| name: Live Web Tests | |
| if: >- | |
| github.event_name != 'pull_request' || | |
| (contains(github.event.pull_request.labels.*.name, 'live-tests') && | |
| (github.event.action != 'labeled' || github.event.label.name == 'live-tests')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| concurrency: | |
| group: ${{ github.workflow }}-live-web-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| steps: | |
| - name: Fail on fork pull requests | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork | |
| run: | | |
| echo "::error::Live tests cannot run on pull requests from forks because GitHub withholds repository secrets. Push the branch to this repository to run them." | |
| exit 1 | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| cache-dependency-path: | | |
| go.sum | |
| tools/ci/go.sum | |
| tools/goose/go.sum | |
| tools/omnaralint/go.sum | |
| - run: make test-live-web | |
| env: | |
| EXA_API_KEY: ${{ secrets.EXA_API_KEY }} | |
| live: | |
| name: Live Provider Tests | |
| if: >- | |
| github.event_name != 'pull_request' || | |
| (contains(github.event.pull_request.labels.*.name, 'live-tests') && | |
| (github.event.action != 'labeled' || github.event.label.name == 'live-tests')) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| concurrency: | |
| group: ${{ github.workflow }}-live-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| steps: | |
| - name: Fail on fork pull requests | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork | |
| run: | | |
| echo "::error::Live tests cannot run on pull requests from forks because GitHub withholds repository secrets. Push the branch to this repository to run them." | |
| exit 1 | |
| - name: Check live test secrets | |
| id: secrets | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| BL_API_KEY: ${{ secrets.BL_API_KEY }} | |
| BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} | |
| BOXD_API_KEY: ${{ secrets.BOXD_API_KEY }} | |
| DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} | |
| UNIKRAFT_API_KEY: ${{ secrets.UNIKRAFT_API_KEY }} | |
| run: | | |
| configured=false | |
| if [ -z "$OPENAI_API_KEY" ]; then | |
| echo "openai=false" >> "$GITHUB_OUTPUT" | |
| echo "OPENAI_API_KEY is not configured; skipping live OpenAI tests." | |
| else | |
| echo "openai=true" >> "$GITHUB_OUTPUT" | |
| configured=true | |
| fi | |
| if [ -z "$OPENROUTER_API_KEY" ]; then | |
| echo "openrouter=false" >> "$GITHUB_OUTPUT" | |
| echo "OPENROUTER_API_KEY is not configured; skipping live OpenRouter tests." | |
| else | |
| echo "openrouter=true" >> "$GITHUB_OUTPUT" | |
| configured=true | |
| fi | |
| if [ -z "$ANTHROPIC_API_KEY" ]; then | |
| echo "anthropic=false" >> "$GITHUB_OUTPUT" | |
| echo "ANTHROPIC_API_KEY is not configured; skipping live Anthropic tests." | |
| else | |
| echo "anthropic=true" >> "$GITHUB_OUTPUT" | |
| configured=true | |
| fi | |
| if { [ -n "$BL_API_KEY" ] && [ -n "$BL_WORKSPACE" ]; } || \ | |
| [ -n "$BOXD_API_KEY" ] || [ -n "$DAYTONA_API_KEY" ] || [ -n "$UNIKRAFT_API_KEY" ]; then | |
| echo "sandboxes=true" >> "$GITHUB_OUTPUT" | |
| configured=true | |
| else | |
| echo "sandboxes=false" >> "$GITHUB_OUTPUT" | |
| echo "No complete sandbox-provider credentials are configured; skipping live sandbox tests." | |
| fi | |
| echo "configured=$configured" >> "$GITHUB_OUTPUT" | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| if: steps.secrets.outputs.configured == 'true' | |
| - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| if: steps.secrets.outputs.configured == 'true' | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| if: steps.secrets.outputs.configured == 'true' | |
| with: | |
| path: | | |
| ~/go/pkg/mod | |
| ~/.cache/go-build | |
| key: go-${{ runner.os }}-${{ runner.arch }}-live-${{ hashFiles('**/go.mod', '**/go.sum') }} | |
| restore-keys: go-${{ runner.os }}-${{ runner.arch }}-live- | |
| - uses: ./.github/actions/setup-test-db | |
| if: steps.secrets.outputs.openai == 'true' || steps.secrets.outputs.openrouter == 'true' || steps.secrets.outputs.anthropic == 'true' | |
| - run: make test-live-openai-responses | |
| if: steps.secrets.outputs.openai == 'true' | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| - run: make test-live-openai-chat-completions | |
| if: steps.secrets.outputs.openai == 'true' | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| - run: make test-live-openrouter | |
| if: steps.secrets.outputs.openrouter == 'true' | |
| env: | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| - run: make test-live-anthropic | |
| if: steps.secrets.outputs.anthropic == 'true' | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| - run: make test-live-api-format-switching | |
| if: steps.secrets.outputs.openai == 'true' && steps.secrets.outputs.openrouter == 'true' && steps.secrets.outputs.anthropic == 'true' | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| - run: make test-live-sandbox-providers | |
| if: steps.secrets.outputs.sandboxes == 'true' | |
| env: | |
| BL_API_KEY: ${{ secrets.BL_API_KEY }} | |
| BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} | |
| BOXD_API_KEY: ${{ secrets.BOXD_API_KEY }} | |
| DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} | |
| UNIKRAFT_API_KEY: ${{ secrets.UNIKRAFT_API_KEY }} | |
| - name: Show database logs on failure | |
| if: failure() && (steps.secrets.outputs.openai == 'true' || steps.secrets.outputs.openrouter == 'true' || steps.secrets.outputs.anthropic == 'true') | |
| run: docker compose logs postgres redis | |
| - name: Stop test database | |
| if: always() && (steps.secrets.outputs.openai == 'true' || steps.secrets.outputs.openrouter == 'true' || steps.secrets.outputs.anthropic == 'true') | |
| run: docker compose down -v |