dojo-e2e.yml

name: e2e

on:
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - "integrations/**"
      - "apps/dojo/**"
      - "middlewares/**"
      - "pnpm-lock.yaml"
      - "pnpm-workspace.yaml"
      - ".github/workflows/dojo-e2e.yml"
      - "sdks/python/**"
      - "sdks/typescript/**"
      - "sdks/dotnet/**"
  pull_request:
    branches: [main]
    paths:
      - "apps/dojo/**"
      - "integrations/**"
      - "middlewares/**"
      - "pnpm-lock.yaml"
      - "pnpm-workspace.yaml"
      - ".github/workflows/dojo-e2e.yml"
      - "sdks/python/**"
      - "sdks/typescript/**"
      - "sdks/dotnet/**"

permissions:
  contents: read

# Pinned Python build toolchain — see .github/python-toolchain.env, which is the
# single place these two values are recorded, along with the green run they came
# from. Change one, change them all: the python-toolchain-pins job in
# lint-release-workflows.yml fails if any workflow's literal disagrees with that file.
env:
  UV_VERSION: "0.12.1"
  PYTHON_VERSION: "3.12"

jobs:
  check-generated-files:
    name: dojo / check-generated-files
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          lfs: true
          persist-credentials: false

      - name: Set up Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
        with:
          node-version-file: ".node-version"

      - name: Install pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86  # v6.0.10
        with:
          version: 10.33.4

      # Mirrors the `test:crewai-config` package script; run via npx (no install)
      # to match the generate step below. Guards the CrewAI Flows + Conversational
      # Flows dojo config that ships with this column.
      - name: Validate CrewAI dojo config (Flows + Conversational Flows)
        working-directory: apps/dojo
        run: npx tsx --test src/crewai.test.ts

      - name: Validate agentFilesMapper and regenerate files.json
        working-directory: apps/dojo
        run: pnpm generate-content-json

      - name: Check files.json is up to date
        working-directory: apps/dojo
        run: |
          if git diff --exit-code src/files.json > /dev/null; then
            echo "✅ No changes detected in dojo/src/files.json. Everything is up to date."
          else
            echo "❌ Detected changes in dojo/src/files.json."
            echo ""
            echo "The committed files.json doesn't match what would be generated."
            echo "Please run \`(p)npm run generate-content-json\` in the apps/dojo folder and commit the updated file."
            echo ""
            echo "The detected diff was as follows:"
            echo "::group::Diff for dojo/src/files.json"
            git diff src/files.json
            echo "::endgroup::"
            exit 1
          fi

  event-trace-unit:
    name: dojo / event-trace-unit
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020  # v7.0.0
        with:
          node-version-file: ".node-version"

      - name: Install pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86  # v6.0.10
        with:
          version: 10.33.4

      - name: Install event trace test dependencies
        working-directory: apps/dojo/e2e
        run: pnpm install --frozen-lockfile --ignore-scripts

      - name: Run event trace unit tests
        working-directory: apps/dojo/e2e
        run: pnpm test:event-trace-unit

      - name: Type-check event trace harness
        working-directory: apps/dojo/e2e
        run: pnpm check:event-trace-types

  dojo:
    name: dojo / ${{ matrix.suite }}
    needs: check-generated-files
    runs-on: depot-ubuntu-24.04
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: a2a-middleware
            test_path: tests/a2aMiddlewareTests
            services: ["dojo", "a2a-middleware"]
            wait_on: http://localhost:9999,http-get://localhost:8011/.well-known/agent.json,http-get://localhost:8012/.well-known/agent.json,http-get://localhost:8013/.well-known/agent.json,http-get://localhost:8014/openapi.json
          - suite: adk-middleware
            test_path: tests/adkMiddlewareTests
            services: ["dojo", "adk-middleware"]
            wait_on: http://localhost:9999,tcp:localhost:8010
          - suite: agno
            test_path: tests/agnoTests
            services: ["dojo", "agno"]
            wait_on: http://localhost:9999,tcp:localhost:8002
          - suite: crew-ai
            test_path: tests/crewAITests
            services: ["dojo", "crew-ai"]
            wait_on: http://localhost:9999,tcp:localhost:8003
          - suite: crewai-conversational-flows
            test_path: tests/crewAIConversationalFlowsTests
            services: ["dojo", "crew-ai"]
            wait_on: http://localhost:9999,tcp:localhost:8003
          - suite: langroid
            test_path: tests/langroidTests
            services: ["dojo", "langroid"]
            wait_on: http://localhost:9999,tcp:localhost:8021
          - suite: langgraph-python
            test_path: tests/langgraphPythonTests
            services: ["dojo", "langgraph-platform-python"]
            wait_on: http://localhost:9999,tcp:localhost:8005
          - suite: langgraph-typescript
            test_path: tests/langgraphTypescriptTests
            services: ["dojo", "langgraph-platform-typescript"]
            wait_on: http://localhost:9999,tcp:localhost:8006
          - suite: langgraph-fastapi
            test_path: tests/langgraphFastAPITests
            services: ["dojo", "langgraph-fastapi"]
            wait_on: http://localhost:9999,tcp:localhost:8004
          - suite: llama-index
            test_path: tests/llamaIndexTests
            services: ["dojo", "llama-index"]
            wait_on: http://localhost:9999,tcp:localhost:8007
          - suite: mastra
            test_path: tests/mastraTests
            services: ["dojo", "mastra"]
            wait_on: http://localhost:9999,tcp:localhost:8008
          - suite: mastra-agent-local
            test_path: tests/mastraAgentLocalTests
            services: ["dojo"]
            wait_on: http://localhost:9999
          - suite: middleware-starter
            test_path: tests/middlewareStarterTests
            services: ["dojo"]
            wait_on: http://localhost:9999
          - suite: pydantic-ai
            test_path: tests/pydanticAITests
            services: ["dojo", "pydantic-ai"]
            wait_on: http://localhost:9999,tcp:localhost:8009
          - suite: server-starter
            test_path: tests/serverStarterTests
            services: ["dojo", "server-starter"]
            wait_on: http://localhost:9999,tcp:localhost:8000
          - suite: server-starter-all
            test_path: tests/serverStarterAllFeaturesTests
            services: ["dojo", "server-starter-all"]
            wait_on: http://localhost:9999,tcp:localhost:8001
          - suite: aws-strands
            test_path: tests/awsStrandsTests
            services: ["dojo", "aws-strands"]
            wait_on: http://localhost:9999,tcp:localhost:8017
          - suite: aws-strands-typescript
            test_path: tests/awsStrandsTypescriptTests
            services: ["dojo", "aws-strands-typescript"]
            wait_on: http://localhost:9999,tcp:localhost:8022
          - suite: claude-agent-sdk-python
            test_path: tests/claudeAgentSdkPythonTests
            services: ["dojo", "claude-agent-sdk-python"]
            wait_on: http://localhost:9999,tcp:localhost:8019
          - suite: claude-agent-sdk-typescript
            test_path: tests/claudeAgentSdkTypescriptTests
            services: ["dojo", "claude-agent-sdk-typescript"]
            wait_on: http://localhost:9999,tcp:localhost:8020
          - suite: ag2
            test_path: tests/ag2Tests
            services: ["dojo", "ag2"]
            wait_on: http://localhost:9999,tcp:localhost:8018
          - suite: microsoft-agent-framework-python
            test_path: tests/microsoftAgentFrameworkPythonTests
            services: ["dojo", "microsoft-agent-framework-python"]
            wait_on: http://localhost:9999,tcp:localhost:8015
          - suite: microsoft-agent-framework-dotnet
            test_path: tests/microsoftAgentFrameworkDotnetTests
            services: ["dojo", "microsoft-agent-framework-dotnet"]
            wait_on: http://localhost:9999,tcp:localhost:8016
          - suite: ag-ui-dotnet
            test_path: tests/agUiDotnetTests
            services: ["dojo", "ag-ui-dotnet"]
            wait_on: http://localhost:9999,tcp:localhost:8023
          # - suite: vercel-ai-sdk
          #   test_path: tests/vercelAISdkTests
          #   services: ["dojo"]
          #   wait_on: http://localhost:9999

    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          lfs: true
          persist-credentials: false

      - name: Detect fork PR
        id: fork-check
        run: |
          if [[ "${{ github.event_name }}" == "pull_request" && \
                "${GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME}" != "${{ github.repository }}" ]]; then
            echo "prefix=fork-" >> "$GITHUB_OUTPUT"
          else
            echo "prefix=" >> "$GITHUB_OUTPUT"
          fi
        env:
          GITHUB_EVENT_PULL_REQUEST_HEAD_REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Set up Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version-file: ".node-version"

      - name: Install pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
        with:
          version: 10.33.4

      # Now that pnpm is available, cache its store to speed installs
      - name: Resolve pnpm store path
        id: pnpm-store
        run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_ENV"

      - name: Cache pnpm store
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-pnpm-store-

      - name: Cache Python dependencies
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry
            ~/.cache/uv
            **/.venv
          # The toolchain segments keep a venv built by one uv/Python from being
          # restored into a job expecting another — including via restore-keys,
          # which would otherwise reach past them on a key miss. Note the limit:
          # this path also captures Poetry-built venvs, and Poetry is installed at
          # `version: latest` below, so a Poetry upgrade is NOT reflected in the key.
          key: ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-pydeps-py${{ env.PYTHON_VERSION }}-uv${{ env.UV_VERSION }}-${{ matrix.suite }}-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }}
          restore-keys: |
            ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-pydeps-py${{ env.PYTHON_VERSION }}-uv${{ env.UV_VERSION }}-${{ matrix.suite }}-
            ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-pydeps-py${{ env.PYTHON_VERSION }}-uv${{ env.UV_VERSION }}-

      - name: Cache Next.js build
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: ${{ github.workspace }}/apps/dojo/.next/cache
          key: ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ hashFiles('apps/dojo/src/**/*.ts', 'apps/dojo/src/**/*.tsx', 'apps/dojo/src/**/*.js', 'apps/dojo/src/**/*.jsx') }}
          restore-keys: |
            ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-

      - name: Install Poetry
        uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263  # v1.4.2
        with:
          version: latest
          virtualenvs-create: true
          virtualenvs-in-project: true

      # uv itself is only invoked indirectly here — the example apps under
      # integrations/*/python/examples are synced by
      # apps/dojo/scripts/prep-dojo-everything.js. Those syncs are deliberately
      # not frozen (the script is shared with local dev, where relocking is
      # wanted), so this pin is what keeps the interpreter and resolver stable
      # across e2e runs.
      - name: Install uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
        with:
          version: ${{ env.UV_VERSION }}
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Set up .NET SDK
        if: ${{ contains(join(matrix.services, ','), 'microsoft-agent-framework-dotnet') || contains(join(matrix.services, ','), 'ag-ui-dotnet') }}
        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68  # v6.0.0
        with:
          dotnet-version: |
            9.0.x
            10.0.x

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Prepare dojo for e2e
        working-directory: apps/dojo
        if: ${{ join(matrix.services, ',') != '' }}
        run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }}
      - name: Cache Playwright browsers
        id: cache-playwright
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
        with:
          path: ~/.cache/ms-playwright
          key: ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-playwright-${{ hashFiles('apps/dojo/e2e/package.json') }}
          restore-keys: |
            ${{ steps.fork-check.outputs.prefix }}${{ runner.os }}-playwright-

      - name: Install e2e dependencies
        working-directory: apps/dojo/e2e
        run: pnpm install --ignore-scripts

      # Neither --with-deps nor install-deps: both shell out to apt, and apt on the
      # runners cannot always reach azure.archive.ubuntu.com — it can retry for long
      # enough to burn this job's whole timeout before a test runs. Chromium's system
      # libraries are already present on the Ubuntu runner image, so downloading the
      # browser is all this job needs.
      #
      # No cache-hit guard either. actions/cache only reports cache-hit: true on an
      # exact key match, so any change to apps/dojo/e2e/package.json restores the
      # browsers via restore-keys and still reports a miss. On such a restore the
      # browsers are already in ~/.cache/ms-playwright and this returns in seconds.
      - name: Install Playwright browsers
        working-directory: apps/dojo/e2e
        run: pnpm exec playwright install chromium

      - name: Create langgraph stub .env files
        if: ${{ contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }}
        run: |
          # langgraph.json declares "env": ".env" — the CLI requires this file
          # to exist on disk. Values don't matter since run-dojo-everything.js
          # injects LLMock env vars into the process environment.
          touch integrations/langgraph/python/examples/.env
          touch integrations/langgraph/typescript/examples/.env

      - name: write langroid env files
        working-directory: integrations/langroid/python/examples
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        # Only inject OPENAI_API_KEY for same-repo PRs / push events.
        # Fork PRs skip this step (and the langroid suite that depends on it)
        # to prevent secret exfiltration via a malicious PR modifying the
        # langroid test runner. Tracked for follow-up: move langroid e2e to a
        # workflow_run pattern so fork PRs can run with secrets injected in
        # the trusted base context.
        if: >-
          contains(join(matrix.services, ','), 'langroid')
          && (github.event_name != 'pull_request'
              || github.event.pull_request.head.repo.full_name == github.repository)
        run: |
          echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > .env

      - name: Run dojo+agents
        uses: JarvusInnovations/background-action@1f5e5fa2462e48a95f97b54b4842d076d1008e3b  # v2.0.0
        if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }}
        env:
          # Backend tool-rendering demos call the live open-meteo API, which
          # rate-limits CI's shared egress IPs and hangs the e2e tests. Return
          # canned weather data instead so these suites are deterministic.
          AG_UI_MOCK_WEATHER: "1"
        with:
          run: |
            node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }}
          working-directory: apps/dojo/e2e
          wait-on: ${{ matrix.wait_on }}
          wait-for: 300000

      - name: Run tests – ${{ matrix.suite }}
        working-directory: apps/dojo/e2e
        env:
          BASE_URL: http://localhost:9999
          PLAYWRIGHT_SUITE: ${{ matrix.suite }}
        run: |
          pnpm test -- ${{ matrix.test_path }}

      - name: Upload traces – ${{ matrix.suite }}
        if: always() # Uploads artifacts even if tests fail
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ matrix.suite }}-playwright-traces
          path: |
            apps/dojo/e2e/test-results/${{ matrix.suite }}/**/*
            apps/dojo/e2e/playwright-report/**/*
          retention-days: 7