.gitlab-ci.yml

# GitLab CI for the LH-Radiology monorepo.
#
# Layers GitLab's managed security scanning (SAST + Secret Detection) on top of the
# project's own validate/test jobs. The Security templates define `sast` and
# `secret_detection`; we only pin their stages below.
stages:
  - validate
  - test
  - secret-detection
  - build

include:
  - template: Security/SAST.gitlab-ci.yml
  - template: Security/Secret-Detection.gitlab-ci.yml
  # Image build and publish lane (#97). Lives in its own file so its jobs' rules:changes
  # can key on that file alone: editing a test lane here must not rebuild ten images.
  - local: ci/images.gitlab-ci.yml

variables:
  SECRET_DETECTION_ENABLED: 'true'

# Run ONE pipeline per change: an MR pipeline for branches with an open MR, and a branch
# pipeline for the default branch. Prevents duplicate push+MR pipelines and — together with
# the shared `.default-rules` below — makes the full test stage run in MR pipelines (not just
# on the default branch), so an MR's green check actually means the suite passed. See #32.
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    # Release tags (vX.Y.Z) get a pipeline solely for the image publish lane (#97). Every
    # other job keys off .default-rules, which matches neither condition, so a tag pipeline
    # runs only the image jobs. Tags are cut from main, which already ran the full suite.
    - if: '$CI_COMMIT_TAG'

# A job that extends this runs in MR pipelines AND on the default branch. Jobs without rules
# default to branch/tag pipelines only and are silently skipped in merge_request_event
# pipelines — which is exactly what let MRs !2/!3 go green while running only validate-contracts.
.default-rules:
  rules: &default-rules
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

validate-contracts:
  extends: .default-rules
  stage: validate
  image: python:3.11-slim
  script:
    - pip install jsonschema pyyaml
    - python scripts/validate_contracts.py

agent-version-bumps:
  # The #129 gate. #124 stopped the card and the handler DISAGREEING; its own comment in
  # validate_contracts.py records what it cannot do -- "it cannot catch both being stale
  # together ... when to bump is a discipline no gate here enforces". This is that gate.
  #
  # MR pipelines ONLY, and not by preference: whether behaviour changed is a fact about a
  # CHANGE, so it needs a base ref. On the default branch there is nothing to compare against.
  #
  # GIT_DEPTH 0 because the default shallow clone can cut above the merge base, and the
  # real-history tests walk back further still; git itself is installed because the script and
  # the tests shell out to it, and python:3.11-slim carries no git (the runner's clone uses the
  # helper image, not this one).
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
  stage: validate
  image: python:3.11-slim
  variables:
    GIT_DEPTH: 0
  script:
    - apt-get update -qq && apt-get install -y -qq --no-install-recommends git
    - pip install pytest
    - python -m pytest scripts/tests -q
    - python scripts/check_agent_version_bumps.py "$CI_MERGE_REQUEST_DIFF_BASE_SHA"

agent-tests:
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e libs/radagent-common pytest pytest-asyncio
    # Each agent is a standalone root; run its tests from inside its dir.
    - |
      for a in agents/*/; do
        echo "== $a ==" && (cd "$a" && python -m pytest -q) || exit 1
      done

worklist-api-tests:
  # The worklist-api python suite ran in NO lane before the #79 ack surface landed here (the
  # ohif-extension lane covers only the extension's vitest; agent-tests loops agents/*/ only).
  # Same rationale as the ris-poller explicit list: a guard CI never runs is not a guard.
  #
  # WORKLIST_STORE_PATH and WORKLIST_FINDINGS_STORE_PATH: main.py builds the app at import time
  # and BOTH of its sqlite stores default under /var/lib, so collection itself needs two writable
  # paths -- true for any environment that runs this suite. This lane used to set only the first
  # and stayed green because the container runs as root and can create /var/lib/lhrad; a normal
  # user hit a PermissionError at collection instead (#134). tests/conftest.py now defaults both
  # to a temp dir when unset, so these are belt-and-braces: explicit here so the lane does not
  # depend on the conftest, and so the next store added to main.py is named in the same place.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  variables:
    WORKLIST_STORE_PATH: /tmp/worklist-tests.sqlite
    WORKLIST_FINDINGS_STORE_PATH: /tmp/worklist-findings-tests.sqlite
  script:
    - pip install -e libs/radagent-common pytest pytest-asyncio fastapi httpx
    - cd integrations/worklist-api && python -m pytest -q

walking-skeleton:
  # The whole-pipeline check CLAUDE.md offers ran in NO lane. agent-tests loops agents/*/, and
  # nothing pointed at mocks/ -- so the harness that #118 found reporting green over a hop that
  # failed five times out of five, and that #125 found replaying one scenario five times, was
  # never executed by CI at all. Same "a guard CI never runs is not a guard" rationale as the
  # worklist-api and mimic-etl lanes.
  #
  # Two things run here, and they check different halves. The suite pins that the five fixtures
  # stay DIFFERENTIATED (#125); the skeleton run itself is the end-to-end contract check and, per
  # !168, exits non-zero on a failed channel instead of printing "All hops validated" over it.
  #
  # No EHR_INBOX_WRITE_ENABLED: the flag-off default is the path most contributors run, and
  # _DemoFhir mirrors the real client by short-circuiting with zero I/O when it is unset (#118).
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e libs/radagent-common pytest pytest-asyncio
    - python -m pytest mocks/tests -q
    - python mocks/run_walking_skeleton.py

mimic-etl-tests:
  # The MIMIC-CXR ETL tooling (scripts/mimic, #68/#76) ran in NO lane before this: agent-tests loops
  # agents/*/, the ris-poller lane lists lib files explicitly, and nothing pointed at scripts/mimic.
  # Same "a guard CI never runs is not a guard" rationale as the worklist-api lane -- the #76
  # referring-physician orderer-seeding logic (referrers.assign + load_study wiring) and the existing
  # manifest/curate/dicom-fixup guards had no CI until now.
  #
  # Slim deps: boto3 (fetch) and pymysql (omrs_client DB paths) are imported LAZILY inside functions,
  # so collection + the pure-logic suite need only pydicom (dicom_fixup module + its test) and httpx
  # (omrs_client import); numpy comes along for pydicom's dataset handling. Tests run from
  # scripts/mimic because each file inserts that dir on sys.path to import the ETL modules by name.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install pytest pydicom numpy httpx
    - cd scripts/mimic && python -m pytest tests -q

a2a-factory-import:
  # Issue #2 acceptance #3: the app extras install and radagent_common.a2a imports cleanly
  # against the pinned a2a-sdk. (agent-tests deliberately runs WITHOUT this extra to keep
  # proving golden-rule 4 — handlers never import a2a.*.)
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e "libs/radagent-common[a2a]"
    - python -c "import radagent_common.a2a; print('radagent_common.a2a import OK')"

a2a-client-it:
  # Issue #7 acceptance: the real A2A client round-trips a skill call against a LIVE mock
  # agent (uvicorn serving the factory app). Needs the a2a extra + uvicorn, so it runs in its
  # own lane rather than in agent-tests.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e "libs/radagent-common[a2a]" uvicorn pytest pytest-asyncio
    - cd libs/radagent-common && python -m pytest tests/test_client.py tests/test_push_roundtrip.py -q

comms-roundtrip-it:
  # Issue #17 acceptance: an orchestrator dispatch round-trips against the LIVE Communications
  # agent. Needs the a2a extra + uvicorn (like a2a-client-it), so it runs in its own lane rather
  # than agent-tests (which deliberately omits the a2a extra to keep proving golden-rule 4).
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e "libs/radagent-common[a2a]" uvicorn pytest pytest-asyncio
    - cd agents/communications && python -m pytest tests/test_roundtrip.py -q

comms-ledger-it:
  # Issue #52: the comms ledger against a LIVE HAPI. Everything else in this pipeline mocks the
  # transport, so it asserts what we SEND -- and passes even when the server rejects every write.
  # Which is what happened: the ledger holds references to resources that live in fhir2 and are
  # absent here (Communication.subject / Task.for -> Patient/*, basedOn -> ServiceRequest/*), HAPI
  # enforces referential integrity on write, and every write came back 400 (HAPI-1094). With
  # integrity merely disabled, HAPI does not INDEX a reference whose target is absent, so the audit
  # searches silently returned 0 rows. Neither failure is expressible in a mock, and both were found
  # only by driving a real image (!41).
  #
  # The fix is one compose setting (auto_create_placeholder_reference_targets). This job is what
  # stops it silently regressing -- via the compose config, the search params, or the reference
  # shapes -- while the mocked lanes stay green. MR 3 (agent) and MR 4 (ack timer) both write here.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  services:
    # Same image and settings as the `comms-ledger` service in docker-compose.yml. Passed as
    # SPRING_APPLICATION_JSON rather than compose's dotted `hapi.fhir.*` keys because GitLab CI
    # variable names cannot contain dots; Spring binds it to the identical properties (verified
    # against the real image -- the suite is 7/7 under both shapes, and 5/7 RED without the
    # placeholder setting, which is the regression this lane exists to catch).
    - name: hapiproject/hapi:v7.4.0
      alias: comms-ledger
      variables:
        SPRING_APPLICATION_JSON: '{"hapi":{"fhir":{"fhir_version":"R4","allow_external_references":true,"auto_create_placeholder_reference_targets":true}}}'
  variables:
    COMMS_LEDGER_BASE_URL: "http://comms-ledger:8080/fhir"
  script:
    # Core lib only: the ledger client needs httpx + pydantic, no a2a/otel/temporal extras.
    - pip install -e libs/radagent-common pytest
    # HAPI takes ~a minute to migrate its schema and serve /metadata. Poll it, and FAIL if it never
    # comes up -- a job that proceeds to a suite it cannot reach would just skip and go green.
    - |
      python - <<'PY'
      import sys, time, urllib.request
      url = "http://comms-ledger:8080/fhir/metadata"
      for i in range(60):
          try:
              with urllib.request.urlopen(url, timeout=5) as r:
                  if r.status == 200:
                      print(f"comms-ledger ready after ~{i * 5}s")
                      sys.exit(0)
          except Exception:
              pass
          time.sleep(5)
      sys.exit("comms-ledger never became healthy after 300s; the integration lane cannot run")
      PY
    - cd libs/radagent-common && python -m pytest tests/test_comms_ledger_it.py -q

ris-poller-tests:
  # Issue #12: fhir2 finalized-report query + the ingress report->workflow poller. Needs the
  # orchestrator deps (temporalio + fastapi); ingress imports radagent_common.client, so the
  # a2a extra comes along too. Run from repo root so `orchestrator` imports as a package.
  #
  # The FHIR test files are named EXPLICITLY rather than running the whole tests/ dir, because
  # the other files there need extras this job does not install. That means a new test file is
  # invisible to CI until it is added HERE -- so #52's comms-ledger + R4-model tests are listed
  # below. A guard CI never runs is not a guard. test_negation.py (#78) is listed for the
  # same reason: 50 tests guarding the critical-finding scanners' negation window -- the
  # "never silence a pneumothorax" pins -- ran in NO lane before this line.
  #
  # THE LIST IS THE COVERAGE, and it drifted anyway. Five files (52 tests) arrived after that
  # note and ran in NO lane at all -- the largest of them the OrthancStableStudyEvent pins, the
  # contract that STARTS every workflow. Add a file to libs/radagent-common/tests and you add it
  # HERE, or to a lane carrying the extras it needs: test_push_roundtrip.py imports uvicorn and
  # starlette at module scope, so it lives in a2a-client-it and would ERROR at collection here.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    # [otel] too (#28): test_worker_tracing.py pins the tracing wiring — the double-span
    # interceptor and the silently-unexported ingress spans. Without the extra the whole module
    # importorskips, and CI would go green on exactly the bugs it exists to catch.
    #
    # test_orthanc_client.py is listed for the same reason (#62). Ingress now reads the study
    # description back from Orthanc, but the ingress seam test injects a fake client — so these
    # are the ONLY tests that exercise the real get_study_description, and without them CI would
    # stay green if the method were deleted. (Needs no extra deps: httpx is a core dep and the
    # file drives async via asyncio.run, not pytest-asyncio.)
    # [imaging] too (#71): test_imaging.py pins the DICOM decode guards (MONOCHROME1 inversion,
    # rescale, malformed-DS, NotAnImage) that every pixel read rides on. Without the extra the
    # module importorskips pydicom and CI goes green on exactly the mutations it exists to catch.
    - pip install -e "libs/radagent-common[a2a,otel,imaging]" "temporalio==1.29.0" fastapi pytest
    - >
      python -m pytest
      libs/radagent-common/tests/test_fhir_client.py
      libs/radagent-common/tests/test_fhir_client_notification.py
      libs/radagent-common/tests/test_ack_link.py
      libs/radagent-common/tests/test_negation.py
      libs/radagent-common/tests/test_openmrs_rest.py
      libs/radagent-common/tests/test_openmrs_rest_reason.py
      libs/radagent-common/tests/test_fhir_models.py
      libs/radagent-common/tests/test_comms_ledger.py
      libs/radagent-common/tests/test_orthanc_client.py
      libs/radagent-common/tests/test_orthanc_client_write.py
      libs/radagent-common/tests/test_imaging.py
      libs/radagent-common/tests/test_tracing.py
      libs/radagent-common/tests/test_presign_concept_drift.py
      libs/radagent-common/tests/test_correlation_ids.py
      libs/radagent-common/tests/test_notification_overview_gp.py
      libs/radagent-common/tests/test_orthanc_stable_event.py
      libs/radagent-common/tests/test_worklist_client.py
      orchestrator/tests -q

orchestrator-e2e-it:
  # Issue #9 / #19: the orchestrator runs FULLY ON MOCKS end-to-end. Boots all six agents as
  # live A2A servers and drives StudyWorkflow start->ARCHIVED on a real Temporal (test server).
  # Needs the a2a extra + uvicorn + temporalio, so it runs in its own lane; the poller lane skips
  # this test (no uvicorn there) via importorskip. This is the "runs fully on mocks in CI" proof.
  extends: .default-rules
  stage: test
  image: python:3.11-slim
  script:
    - pip install -e "libs/radagent-common[a2a]" uvicorn "temporalio==1.29.0" pytest
    - python -m pytest orchestrator/tests/test_e2e_temporal.py -q

ohif-extension-tests:
  # Issue #21: the OHIF worklist extension's vitest suite + typecheck. Runs against the
  # extension's own lockfile, no OHIF monorepo needed. Without this lane the extension's
  # 30 tests never run in CI, and a guard CI never runs is not a guard. The Docker image
  # build (which pulls the OHIF monorepo) stays a local/deploy concern.
  extends: .default-rules
  stage: test
  image: node:20-bookworm-slim
  script:
    - cd integrations/ohif-extension
    - npm ci
    - npm test
    - npm run typecheck

# The managed SAST/Secret-Detection templates default to branch pipelines only, so with the
# workflow block above (which suppresses the push pipeline on MR branches) they would run on
# NO merge request at all. Opt the concrete analyzer jobs into MR pipelines by overriding
# ONLY their `rules`. Two rules of thumb (learned the hard way, see #32):
#   - Do NOT use `extends: .default-rules`: `extends` replaces the template's own base, which
#     drops the analyzer image and breaks `/analyzer run`.
#   - Do NOT put `rules` on the `sast` job: it is a config-only placeholder whose template
#     `rules: when: never` must stand, or it runs its `exit 1` script. Target `semgrep-sast`.
sast:
  stage: test        # config-only: sets the analyzer stage; the placeholder never runs itself

semgrep-sast:
  rules: *default-rules

secret_detection:
  stage: secret-detection
  rules: *default-rules