Merge pull request #4 from dormitivegit/release/0.4.0rc1 #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Software Evidence Controls Informational Dogfood | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| dogfood: | |
| name: Published-verifier informational dogfood | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository with base history | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.11" | |
| - name: Install the published Software Evidence Controls verifier | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| evidence_dir="$RUNNER_TEMP/software-evidence-controls-dogfood-evidence" | |
| mkdir -p "$evidence_dir" | |
| python -m pip install --disable-pip-version-check software-evidence-controls==0.4.0rc1 | |
| installed_version="$(python -c 'from importlib.metadata import version; print(version("software-evidence-controls"))')" | |
| test "$installed_version" = "0.4.0rc1" | |
| { | |
| software-evidence-controls --version | |
| printf 'python-distribution-version: %s\n' "$installed_version" | |
| } > "$evidence_dir/identity.txt" | |
| - name: Execute informational base-to-head dogfood | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| EVENT_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| EVENT_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| EVENT_PUSH_BEFORE: ${{ github.event.before }} | |
| EVENT_GITHUB_SHA: ${{ github.sha }} | |
| run: | | |
| set -euo pipefail | |
| evidence_dir="$RUNNER_TEMP/software-evidence-controls-dogfood-evidence" | |
| manifest="$RUNNER_TEMP/software-evidence-controls-task236-base-manifest.jsonl" | |
| workspace_path="$GITHUB_WORKSPACE" | |
| event="$EVENT_NAME" | |
| mkdir -p "$evidence_dir" | |
| case "$event" in | |
| pull_request) | |
| base_sha="$EVENT_PR_BASE_SHA" | |
| head_sha="$EVENT_PR_HEAD_SHA" | |
| ;; | |
| push) | |
| base_sha="$EVENT_PUSH_BEFORE" | |
| head_sha="$EVENT_GITHUB_SHA" | |
| ;; | |
| workflow_dispatch) | |
| head_sha="$EVENT_GITHUB_SHA" | |
| base_sha="$(git show -s --format=%P "$head_sha" | awk '{print $1}')" | |
| ;; | |
| *) | |
| printf 'unsupported event: %s\n' "$event" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| if [[ -z "$head_sha" ]] || [[ -z "$base_sha" ]] || [[ "$base_sha" =~ ^0{40}$ ]]; then | |
| python - "$evidence_dir/decision.json" "$event" "$base_sha" "$head_sha" <<'PY' | |
| import json | |
| import sys | |
| path, event, base_sha, head_sha = sys.argv[1:] | |
| payload = { | |
| "task_id": "TASK236", | |
| "mode": "informational_nonblocking", | |
| "event": event, | |
| "base_sha": base_sha, | |
| "head_sha": head_sha, | |
| "tool": "software-evidence-controls==0.4.0rc1", | |
| "positive_control_pass": False, | |
| "canary": {"landed_proven": False, "ci10_observed": False, "process_exit": None}, | |
| "ordinary_tests": {"executed": False, "exit": None, "count": None}, | |
| "head_observation": { | |
| "structured_json_parsed": False, | |
| "process_exit": None, | |
| "result": "BASE_UNAVAILABLE", | |
| "finding_codes": [], | |
| "finding_severities": [], | |
| }, | |
| "observed_decision": "BASE_UNAVAILABLE", | |
| "human_review_required": True, | |
| "same_workspace_path_used": True, | |
| "enforced": False, | |
| } | |
| with open(path, "w", encoding="utf-8") as handle: | |
| json.dump(payload, handle, sort_keys=True, indent=2) | |
| handle.write("\n") | |
| PY | |
| printf '%s\n' 'SOFTWARE_EVIDENCE_CONTROLS_TASK236=BASE_UNAVAILABLE' >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| git cat-file -e "$base_sha^{commit}" | |
| git cat-file -e "$head_sha^{commit}" | |
| git checkout --detach "$base_sha" | |
| test "$(pwd -P)" = "$workspace_path" | |
| base_tree="$(git rev-parse "$base_sha^{tree}")" | |
| source_path="$workspace_path/src" | |
| test -d "$source_path" | |
| software-evidence-controls corpus freeze "$source_path" --manifest "$manifest" --format json > /dev/null | |
| manifest_sha256="$(sha256sum "$manifest" | awk '{print $1}')" | |
| set +e | |
| software-evidence-controls corpus verify "$manifest" --accepted-manifest-sha256 "$manifest_sha256" --expected-root "$source_path" --detect-new --format json > "$evidence_dir/base-positive.json" | |
| base_positive_exit=$? | |
| set -e | |
| python - "$evidence_dir/base-positive.json" "$base_positive_exit" <<'PY' | |
| import json | |
| import sys | |
| path, process_exit = sys.argv[1:] | |
| with open(path, encoding="utf-8") as handle: | |
| payload = json.load(handle) | |
| if int(process_exit) != 0 or payload.get("result") != "PASS" or payload.get("findings") != []: | |
| raise SystemExit("Task236 positive control is not clean") | |
| PY | |
| canary_path="$source_path/.software_evidence_controls_task236_canary.py" | |
| expected_canary_sha256="b806ce69af93c9ed1f37bb44b85831cfe94053a53979a5030322288e88e19888" | |
| printf '%s\n' \ | |
| '# Task236 ephemeral runner-only canary' \ | |
| 'TASK236_CANARY = "software-evidence-controls-dogfood"' > "$canary_path" | |
| actual_canary_sha256="$(sha256sum "$canary_path" | awk '{print $1}')" | |
| test "$actual_canary_sha256" = "$expected_canary_sha256" | |
| test -f "$canary_path" | |
| { | |
| printf 'CANARY_PATH=%s\n' "$canary_path" | |
| printf 'CANARY_SHA256=%s\n' "$actual_canary_sha256" | |
| printf 'CANARY_PRESENT=YES\n' | |
| printf '%s\n' 'CANARY_MACHINE_DECISION=ROUTE_TO_HUMAN_REVIEW' | |
| } > "$evidence_dir/canary-proof.txt" | |
| set +e | |
| software-evidence-controls corpus verify "$manifest" --accepted-manifest-sha256 "$manifest_sha256" --expected-root "$source_path" --detect-new --format json > "$evidence_dir/canary.json" | |
| canary_exit=$? | |
| set -e | |
| python - "$evidence_dir/canary.json" "$evidence_dir/canary-decision.json" "$canary_exit" <<'PY' | |
| import json | |
| import sys | |
| path, decision_path, process_exit = sys.argv[1:] | |
| with open(path, encoding="utf-8") as handle: | |
| payload = json.load(handle) | |
| findings = [(item.get("code"), item.get("severity")) for item in payload.get("findings", [])] | |
| if int(process_exit) != 0 or payload.get("result") != "PASS" or findings != [("CI10_NEW_SOURCE_DETECTED", "WARN")]: | |
| raise SystemExit("Task236 canary did not produce its expected structured result") | |
| with open(decision_path, "w", encoding="utf-8") as handle: | |
| json.dump( | |
| { | |
| "observed_decision": "ROUTE_TO_HUMAN_REVIEW", | |
| "human_review_required": True, | |
| "process_exit": int(process_exit), | |
| "finding_codes": ["CI10_NEW_SOURCE_DETECTED"], | |
| }, | |
| handle, | |
| sort_keys=True, | |
| indent=2, | |
| ) | |
| handle.write("\n") | |
| PY | |
| rm -f "$canary_path" | |
| test ! -e "$canary_path" | |
| test -z "$(git status --porcelain)" | |
| git checkout --detach "$head_sha" | |
| test "$(pwd -P)" = "$workspace_path" | |
| head_tree="$(git rev-parse "$head_sha^{tree}")" | |
| set +e | |
| PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src python -m unittest discover -s tests -v > "$evidence_dir/ordinary-tests.log" 2>&1 | |
| ordinary_tests_exit=$? | |
| set -e | |
| ordinary_tests_count="$(sed -nE 's/^Ran ([0-9]+) tests?.*/\1/p' "$evidence_dir/ordinary-tests.log" | tail -n 1)" | |
| failed_test_count="$(grep -c '_FailedTest' "$evidence_dir/ordinary-tests.log" || true)" | |
| set +e | |
| software-evidence-controls corpus verify "$manifest" --accepted-manifest-sha256 "$manifest_sha256" --expected-root "$source_path" --detect-new --format json > "$evidence_dir/head-observation.json" 2> "$evidence_dir/head-observation.stderr" | |
| head_verify_exit=$? | |
| set -e | |
| python - \ | |
| "$evidence_dir/decision.json" \ | |
| "$evidence_dir/head-observation.json" \ | |
| "$event" "$base_sha" "$head_sha" "$base_tree" "$head_tree" "$manifest_sha256" \ | |
| "$base_positive_exit" "$canary_exit" "$ordinary_tests_exit" "$ordinary_tests_count" "$failed_test_count" "$head_verify_exit" <<'PY' | |
| import json | |
| import sys | |
| ( | |
| decision_path, | |
| observation_path, | |
| event, | |
| base_sha, | |
| head_sha, | |
| base_tree, | |
| head_tree, | |
| manifest_sha256, | |
| base_positive_exit, | |
| canary_exit, | |
| ordinary_tests_exit, | |
| ordinary_tests_count, | |
| failed_test_count, | |
| head_verify_exit, | |
| ) = sys.argv[1:] | |
| with open(observation_path, encoding="utf-8") as handle: | |
| observation = json.load(handle) | |
| findings = observation.get("findings") | |
| if not isinstance(findings, list): | |
| raise SystemExit("Task236 head observation is not a ModuleResult JSON payload") | |
| codes = [item.get("code") for item in findings] | |
| severities = [item.get("severity") for item in findings] | |
| if not all(isinstance(code, str) for code in codes) or not all(isinstance(severity, str) for severity in severities): | |
| raise SystemExit("Task236 head observation has malformed findings") | |
| ordinary_count = int(ordinary_tests_count) if ordinary_tests_count else None | |
| ordinary_green = int(ordinary_tests_exit) == 0 and int(failed_test_count) == 0 | |
| if not ordinary_green: | |
| observed_decision = "ORDINARY_TESTS_NONGREEN" | |
| human_review_required = True | |
| elif any(severity in {"ERROR", "HOLD"} for severity in severities): | |
| observed_decision = "BLOCKING_ACCEPTANCE_EVIDENCE" | |
| human_review_required = True | |
| elif "CI10_NEW_SOURCE_DETECTED" in codes: | |
| observed_decision = "ROUTE_TO_HUMAN_REVIEW" | |
| human_review_required = True | |
| elif int(head_verify_exit) != 0 or observation.get("result") != "PASS": | |
| observed_decision = "INVESTIGATE_NONGREEN" | |
| human_review_required = True | |
| else: | |
| observed_decision = "CONTINUE" | |
| human_review_required = False | |
| payload = { | |
| "task_id": "TASK236", | |
| "mode": "informational_nonblocking", | |
| "event": event, | |
| "base_sha": base_sha, | |
| "head_sha": head_sha, | |
| "base_tree": base_tree, | |
| "head_tree": head_tree, | |
| "manifest_sha256": manifest_sha256, | |
| "tool": "software-evidence-controls==0.4.0rc1", | |
| "positive_control_pass": int(base_positive_exit) == 0, | |
| "canary": { | |
| "landed_proven": True, | |
| "ci10_observed": "CI10_NEW_SOURCE_DETECTED" in [ | |
| item.get("code") for item in json.load(open(decision_path.replace("decision.json", "canary.json"), encoding="utf-8")).get("findings", []) | |
| ], | |
| "process_exit": int(canary_exit), | |
| }, | |
| "ordinary_tests": { | |
| "executed": True, | |
| "exit": int(ordinary_tests_exit), | |
| "count": ordinary_count, | |
| "failed_test_count": int(failed_test_count), | |
| }, | |
| "head_observation": { | |
| "structured_json_parsed": True, | |
| "process_exit": int(head_verify_exit), | |
| "result": observation.get("result", ""), | |
| "finding_codes": codes, | |
| "finding_severities": severities, | |
| }, | |
| "observed_decision": observed_decision, | |
| "human_review_required": human_review_required, | |
| "same_workspace_path_used": True, | |
| "enforced": False, | |
| } | |
| with open(decision_path, "w", encoding="utf-8") as handle: | |
| json.dump(payload, handle, sort_keys=True, indent=2) | |
| handle.write("\n") | |
| PY | |
| python - "$evidence_dir/subject.json" "$event" "$base_sha" "$head_sha" "$base_tree" "$head_tree" "$manifest_sha256" <<'PY' | |
| import json | |
| import sys | |
| path, event, base_sha, head_sha, base_tree, head_tree, manifest_sha256 = sys.argv[1:] | |
| payload = { | |
| "event": event, | |
| "base_sha": base_sha, | |
| "head_sha": head_sha, | |
| "base_tree": base_tree, | |
| "head_tree": head_tree, | |
| "manifest_sha256": manifest_sha256, | |
| "same_workspace_path_used": True, | |
| } | |
| with open(path, "w", encoding="utf-8") as handle: | |
| json.dump(payload, handle, sort_keys=True, indent=2) | |
| handle.write("\n") | |
| PY | |
| { | |
| printf '%s\n' '## Software Evidence Controls informational dogfood' | |
| printf '%s\n' "- Event: \`$event\`" | |
| printf '%s\n' "- Base: \`$base_sha\`" | |
| printf '%s\n' "- Head: \`$head_sha\`" | |
| printf '%s\n' "- Manifest SHA-256: \`$manifest_sha256\`" | |
| printf '%s\n' "- Ordinary tests: exit \`$ordinary_tests_exit\`, count \`${ordinary_tests_count:-unparsed}\`" | |
| printf '%s\n' "- Observed decision: \`$(python -c 'import json; print(json.load(open("'"$evidence_dir"'/decision.json", encoding="utf-8"))["observed_decision"])')\`" | |
| printf '%s\n' '- Enforcement: `false` (informational only)' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload bounded dogfood evidence | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: software-evidence-controls-dogfood-evidence | |
| path: ${{ runner.temp }}/software-evidence-controls-dogfood-evidence | |
| if-no-files-found: error |