ci.yml

# KeepKey python-keepkey CI
#
# Builds the 7.14.3 Bitcoin-only emulator from the fork release branch and
# runs the full Python integration suite against that exact product variant.
#
# Stage 1: GATE (seconds)
#   └─ lint        Python syntax + deterministic protocol contract tests
#
# Stage 2: TEST (gated by Stage 1)
#   └─ integration full pytest suite against emulator

name: CI

on:
  push:
    branches: [master, develop, reconcile/upstream-sync, 'feature/**', 'fix/**', 'hotfix/**']
  pull_request:
    branches: [master, develop, reconcile/upstream-sync]

# One run per ref: a new push supersedes the old instead of both burning a
# runner to completion.
concurrency:
  group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  # ═══════════════════════════════════════════════════════════
  # STAGE 1: GATE
  # ═══════════════════════════════════════════════════════════

  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Syntax check
        run: python -m py_compile keepkeylib/*.py

      - name: Install contract-test dependencies
        run: |
          pip install "protobuf>=3.20,<4" mnemonic ecdsa pytest requests

      - name: Run deterministic Zcash PCZT contract tests
        env:
          PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
        run: |
          python -m pytest -q \
            tests/test_msg_zcash_sign_pczt.py \
            tests/test_zcash_seed_fingerprint_helper.py

      - name: Verify offline transaction fixture manifest
        run: python tests/tx_fixture_manifest.py --check

      - name: Lint summary
        run: |
          echo "## 🔑 KeepKey python-keepkey — Lint" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"
          echo "| Check | Status |" >> "$GITHUB_STEP_SUMMARY"
          echo "|-------|--------|" >> "$GITHUB_STEP_SUMMARY"
          echo "| Syntax | ✅ PASS |" >> "$GITHUB_STEP_SUMMARY"
          echo "| Zcash PCZT contract | ✅ PASS |" >> "$GITHUB_STEP_SUMMARY"
          echo "| Offline fixture integrity | ✅ PASS |" >> "$GITHUB_STEP_SUMMARY"
          FIXTURE_SHA=$(sha256sum tests/txcache/manifest.json | cut -d' ' -f1)
          echo "" >> "$GITHUB_STEP_SUMMARY"
          printf 'Fixture manifest SHA-256: `%s`\n' "$FIXTURE_SHA" >> "$GITHUB_STEP_SUMMARY"

  # ═══════════════════════════════════════════════════════════
  # STAGE 2: TEST — build Bitcoin-only emulator, run pytest
  # ═══════════════════════════════════════════════════════════

  integration:
    needs: [lint]
    runs-on: ubuntu-latest
    timeout-minutes: 15

    # NO published emulator image. This job BUILDS one from current firmware.
    #
    # It used to pull kktech/kkemu:latest -- a floating tag whose image was
    # five months and six minor versions stale. That single fact caused every
    # symptom we chased: 80 tests gating on requires_firmware("7.15.0") skipped
    # silently, and one unskipped test drove a ctime() path that segfaults on
    # the old image and does not exist in current firmware.
    #
    # Publishing a fresher image would only reset that clock. Building from
    # source removes the class: the emulator under test is, by construction,
    # the firmware the tests were written against.

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          path: python-keepkey

      # This PR targets the 7.14.3 test line and is pinned by firmware PR #604.
      # Build that product branch so the tests and firmware under review stay
      # on the same compatibility surface.
      - name: Checkout firmware
        uses: actions/checkout@v4
        with:
          repository: BitHighlander/keepkey-firmware
          ref: release/7.14.3-bitcoin-only
          path: keepkey-firmware

      # NOT `submodules: recursive`. trezor-firmware carries a micropython
      # vendor tree whose lib/lwip lives on git.savannah.gnu.org, which serves
      # dumb HTTP and cannot do the shallow clone actions/checkout requests --
      # it fails the whole job. The firmware repo's own CI inits exactly these
      # paths, non-recursively, for the same reason.
      - name: Init the submodules the emulator build needs
        working-directory: keepkey-firmware
        run: |
          git submodule update --init --depth 1 deps/crypto/trezor-firmware
          git submodule update --init --depth 1 deps/device-protocol
          git submodule update --init --depth 1 deps/googletest
          git submodule update --init --depth 1 deps/qrenc/QR-Code-generator
          git submodule update --init --depth 1 deps/sca-hardening/SecAESSTM32

      # Test THIS checkout of python-keepkey, not the one the firmware pins.
      - name: Overlay this python-keepkey onto the firmware tree
        run: |
          rm -rf keepkey-firmware/deps/python-keepkey
          cp -a python-keepkey keepkey-firmware/deps/python-keepkey

      - name: Build the emulator
        timeout-minutes: 20
        working-directory: keepkey-firmware
        run: |
          docker build --build-arg coinsupport=-DKK_BITCOIN_ONLY=ON \
            -t kkemu-ci -f scripts/emulator/Dockerfile .

      - name: Start the emulator
        run: |
          docker run -d --name kkemu \
            -p 11044:11044/udp -p 11045:11045/udp -p 5000:5000 kkemu-ci
          sleep 3
          docker logs kkemu | head -5

      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        working-directory: python-keepkey
        run: |
          pip install --upgrade pip
          pip install "protobuf>=3.20,<4"
          pip install -e .
          pip install pytest pytest-timeout semver rlp requests eth-keys pycryptodome

      - name: Wait for emulator
        run: |
          echo "Waiting for emulator bridge on port 5000..."
          ready=false
          for i in $(seq 1 30); do
            if curl -sf -X POST http://localhost:5000/exchange/main \
                 -H 'Content-Type: application/json' \
                 -d '{"data":""}' > /dev/null 2>&1; then
              echo "Emulator ready after ${i}s"
              ready=true
              break
            fi
            sleep 1
          done
          [ "$ready" = true ] || {
            docker logs kkemu
            echo "FATAL: emulator bridge did not become ready" >&2
            exit 1
          }

      # "The emulator answered a ping" is not "the emulator is the right
      # firmware". CI ran a 7.16-era suite against a 7.10.0 image for five
      # months: 80 tests gate on requires_firmware("7.15.0") and silently
      # SKIPPED, while one unskipped test drove a code path that segfaults in
      # 7.10.0 and is already fixed in 7.15 -- which reads as a product failure
      # but is only a stale image. A floating tag cannot tell you that. This
      # can, and it fails closed.
      - name: Assert the emulator is not older than the suite
        timeout-minutes: 2
        env:
          KK_TRANSPORT_MAIN: "127.0.0.1:11044"
          KK_TRANSPORT_DEBUG: "127.0.0.1:11045"
          KK_MIN_FW: "7.14.3"
          KK_UDP_TIMEOUT: "20"
          KK_EXPECTED_VARIANT: "EmulatorBTC"
        working-directory: keepkey-firmware/deps/python-keepkey/tests
        run: |
          python - <<'PY'
          import os, sys
          sys.path.insert(0, '..')
          import config
          from keepkeylib.client import KeepKeyDebuglinkClient
          c = KeepKeyDebuglinkClient(config.TRANSPORT(*config.TRANSPORT_ARGS,
                                                      **config.TRANSPORT_KWARGS))
          c.set_debuglink(config.DEBUG_TRANSPORT(*config.DEBUG_TRANSPORT_ARGS,
                                                 **config.DEBUG_TRANSPORT_KWARGS))
          c.init_device()
          f = c.features
          got = (f.major_version, f.minor_version, f.patch_version)
          floor = tuple(int(x) for x in os.environ['KK_MIN_FW'].split('.'))
          print('emulator firmware %d.%d.%d, floor %s' %
                (got + (os.environ['KK_MIN_FW'],)))
          if got < floor:
              sys.exit('FATAL: the emulator image predates the tests that run '
                       'against it. Republish kktech/kkemu from current '
                       'firmware and pin the new digest above.')
          if f.firmware_variant != os.environ['KK_EXPECTED_VARIANT']:
              sys.exit('FATAL: expected %s, got firmware variant %r' %
                       (os.environ['KK_EXPECTED_VARIANT'],
                        f.firmware_variant))
          PY

      # Step-level timeout, deliberately: a JOB-level timeout ends the job as
      # "cancelled", which reads as an infra blip. A step timeout is a FAILURE.
      - name: Run integration tests
        timeout-minutes: 8
        env:
          KK_TRANSPORT_MAIN: "127.0.0.1:11044"
          KK_TRANSPORT_DEBUG: "127.0.0.1:11045"
          PYTHONPATH: "${{ github.workspace }}/keepkey-firmware/deps/python-keepkey"
          PYTHONUNBUFFERED: "1"
          # A crashed emulator now raises instead of blocking in recv() forever.
          KK_UDP_TIMEOUT: "45"
        run: |
          # From the OVERLAID copy, not the standalone checkout: the
          # storage-version-gate tests assert against lib/firmware/storage.c,
          # which they find by walking UP. Run them as a sibling of the
          # firmware and they resolve; run them standalone and they fail
          # claiming the sources are missing.
          cd keepkey-firmware/deps/python-keepkey/tests
          python tx_fixture_manifest.py --check
          # Docker DNATs localhost-published packets before the filter OUTPUT
          # chain, so they no longer have `lo` as their output interface. Keep
          # only the emulator's two UDP ports reachable through the offline
          # gate; every other new non-loopback connection remains rejected.
          sudo iptables -I OUTPUT 1 ! -o lo -m conntrack --ctstate NEW -j REJECT
          sudo iptables -I OUTPUT 1 -p udp -m multiport --dports 11044,11045 -j ACCEPT
          cleanup_network_gate() {
            sudo iptables -D OUTPUT -p udp -m multiport --dports 11044,11045 -j ACCEPT
            sudo iptables -D OUTPUT ! -o lo -m conntrack --ctstate NEW -j REJECT
          }
          trap cleanup_network_gate EXIT
          pytest -vv -s --timeout=60 --timeout-method=signal --maxfail=1 --junitxml=junit.xml 2>&1 | tee pytest-output.txt
          echo "${PIPESTATUS[0]}" > status

      - name: Test summary
        if: always()
        run: |
          XML="keepkey-firmware/deps/python-keepkey/tests/junit.xml"
          MANIFEST="keepkey-firmware/deps/python-keepkey/tests/txcache/manifest.json"
          echo "## 🔑 KeepKey python-keepkey — Integration Tests" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"
          if [ -f "$MANIFEST" ]; then
            FIXTURE_SHA=$(sha256sum "$MANIFEST" | cut -d' ' -f1)
            printf 'Fixture manifest SHA-256: `%s`\n' "$FIXTURE_SHA" >> "$GITHUB_STEP_SUMMARY"
            echo "" >> "$GITHUB_STEP_SUMMARY"
          fi

          if [ ! -f "$XML" ]; then
            echo "❌ **No test results found** — suite may have crashed before completion." >> "$GITHUB_STEP_SUMMARY"
          else
            TOTAL=$(grep -oP 'tests="\K[0-9]+' "$XML" | head -1)
            FAILED=$(grep -oP 'failures="\K[0-9]+' "$XML" | head -1)
            ERRORS=$(grep -oP 'errors="\K[0-9]+' "$XML" | head -1)
            SKIPPED=$(grep -oP 'skipped="\K[0-9]+' "$XML" | head -1)
            TIME=$(grep -oP 'time="\K[0-9.]+' "$XML" | head -1)

            TOTAL=${TOTAL:-0}; FAILED=${FAILED:-0}; ERRORS=${ERRORS:-0}; SKIPPED=${SKIPPED:-0}
            PASSED=$((TOTAL - FAILED - ERRORS - SKIPPED))

            if [ "$FAILED" -eq 0 ] && [ "$ERRORS" -eq 0 ]; then
              echo "✅ **$PASSED of $TOTAL TESTS PASSED** in ${TIME}s" >> "$GITHUB_STEP_SUMMARY"
            else
              echo "❌ **$((FAILED + ERRORS)) of $TOTAL TESTS FAILED**" >> "$GITHUB_STEP_SUMMARY"
            fi

            echo "" >> "$GITHUB_STEP_SUMMARY"
            echo "| Metric | Count |" >> "$GITHUB_STEP_SUMMARY"
            echo "|--------|-------|" >> "$GITHUB_STEP_SUMMARY"
            echo "| Total | $TOTAL |" >> "$GITHUB_STEP_SUMMARY"
            echo "| ✅ Passed | $PASSED |" >> "$GITHUB_STEP_SUMMARY"
            echo "| ⏭️ Skipped | $SKIPPED |" >> "$GITHUB_STEP_SUMMARY"
            echo "| ❌ Failed | $FAILED |" >> "$GITHUB_STEP_SUMMARY"
            echo "| 💥 Errors | $ERRORS |" >> "$GITHUB_STEP_SUMMARY"

            # Itemize skipped with reasons
            python3 -c "import xml.etree.ElementTree as ET,sys;tree=ET.parse(sys.argv[1]);[print(f'| \`{tc.get(\"classname\",\"\")}.{tc.get(\"name\",\"\")}\` | {tc.find(\"skipped\").get(\"message\",tc.find(\"skipped\").text or \"No reason given\")} |') for tc in tree.iter('testcase') if tc.find('skipped') is not None]" "$XML" > /tmp/skip_rows.txt 2>/dev/null || true

            if [ -s /tmp/skip_rows.txt ]; then
              echo "" >> "$GITHUB_STEP_SUMMARY"
              echo "### Skipped Tests" >> "$GITHUB_STEP_SUMMARY"
              echo "| Test | Reason |" >> "$GITHUB_STEP_SUMMARY"
              echo "|------|--------|" >> "$GITHUB_STEP_SUMMARY"
              cat /tmp/skip_rows.txt >> "$GITHUB_STEP_SUMMARY"
            fi
          fi

          echo "" >> "$GITHUB_STEP_SUMMARY"
          echo "---" >> "$GITHUB_STEP_SUMMARY"
          echo "*KeepKey python-keepkey CI*" >> "$GITHUB_STEP_SUMMARY"

      # NO check_name. With one, this action publishes a SEPARATE check run
      # via the Checks API, and its require_tests default of 'false' means an
      # absent junit.xml -- which is exactly what a killed pytest leaves behind
      # -- reports conclusion:success with zero duration. That green check sat
      # on top of a job timing out at 30 minutes for at least six merges.
      # annotate_only keeps the inline annotations without minting a check.
      - name: Annotate test results
        uses: mikepenz/action-junit-report@v4
        if: always()
        with:
          report_paths: keepkey-firmware/deps/python-keepkey/tests/junit.xml
          annotate_only: true
          require_tests: true
          fail_on_failure: true

      - name: Fail on test failure
        if: always()
        run: |
          STATUS=$(cat keepkey-firmware/deps/python-keepkey/tests/status 2>/dev/null || echo "1")
          [ "$STATUS" = "0" ] || exit 1