openfirma/.github/workflows/e2e-tests.yml at main · Firma-AI/openfirma · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
name: E2E Tests

on:
  pull_request:
    types: [opened, reopened, synchronize]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: e2e-tests-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  package:
    name: Package release workspace
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (github.event.pull_request.head.repo.full_name == github.repository &&
      github.event.pull_request.user.type == 'Bot' &&
      contains(github.event.pull_request.labels.*.name, 'release') &&
      startsWith(github.event.pull_request.head.ref, 'release-plz-'))
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
        with:
          rustflags: ""
          cache: false

      - name: Load tool versions
        shell: bash
        run: grep -E '^[A-Z0-9_]+=' tool-versions.env >> "$GITHUB_ENV"

      - name: Install protoc
        uses: ./.github/actions/setup-protoc
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      # cargo package assembles and verifies packages without publishing them.
      - name: Package workspace without publishing
        run: cargo package --workspace --locked

  live-agent:
    name: live-agent (${{ matrix.os }}, ${{ matrix.agent.name }})
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (github.event.pull_request.head.repo.full_name == github.repository &&
      github.event.pull_request.user.type == 'Bot' &&
      contains(github.event.pull_request.labels.*.name, 'release') &&
      startsWith(github.event.pull_request.head.ref, 'release-plz-'))
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
        agent:
          - name: claude
            package: "@anthropic-ai/claude-code"
          - name: codex
            package: "@openai/codex"

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
        with:
          rustflags: ""
          cache: false

      - name: Install cargo-binstall
        uses: cargo-bins/cargo-binstall@75b4bfae1b2c753a6806bbce6e6cb89b602de33c # v1.22.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Load tool versions
        shell: bash
        run: grep -E '^[A-Z0-9_]+=' tool-versions.env >> "$GITHUB_ENV"
      - name: Install cargo-nextest
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: command -v cargo-nextest || cargo binstall -y --force --locked cargo-nextest@$CARGO_NEXTEST_VERSION
        shell: bash

      - name: Install protoc
        uses: ./.github/actions/setup-protoc
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install bubblewrap (Linux)
        if: runner.os == 'Linux'
        run: sudo apt-get install -y bubblewrap

      # Ubuntu 24.04 ships kernel.apparmor_restrict_unprivileged_userns=1, which
      # transitions bwrap to a profile that strips CAP_NET_ADMIN inside its user
      # namespace, so it cannot bring up loopback (RTM_NEWADDR). Install the
      # targeted AppArmor profile that lets bwrap keep its caps in the userns.
      - name: Allow bwrap user namespaces via AppArmor profile (Linux)
        if: runner.os == 'Linux'
        run: |
          sudo tee /etc/apparmor.d/bwrap >/dev/null <<'EOF'
          abi <abi/4.0>,
          include <tunables/global>

          profile bwrap /usr/bin/bwrap flags=(unconfined) {
              userns,
              include if exists <local/bwrap>
          }
          EOF
          sudo apparmor_parser -r /etc/apparmor.d/bwrap

      - name: Install ${{ matrix.agent.name }}
        run: |
          npm install -g '${{ matrix.agent.package }}'
          ${{ matrix.agent.name }} --version

      - name: Authenticate codex
        if: matrix.agent.name == 'codex'
        run: printenv OPENAI_API_KEY | codex login --with-api-key
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

      # nextest builds the firma binary as part of the live-agent test; firma_bin()
      # reads its path from CARGO_BIN_EXE_firma.
      - name: Run live-agent tests
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          FIRMA_RUN_VZ_STRUCTURAL_NETWORK: '1'
        run: cargo nextest run -p firma --test live-agent --run-ignored all --no-tests=fail -E 'test(/${{ matrix.agent.name }}::/)'