Add cross-model epistemic conformance benchmark #100
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Phase 9 Empirical Cold Consumer | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| model: | |
| description: Ollama model tag to evaluate | |
| required: false | |
| default: qwen2.5:3b | |
| num_ctx: | |
| description: Ollama context length | |
| required: false | |
| default: '32768' | |
| num_predict: | |
| description: Maximum generated tokens per Ollama pass | |
| required: false | |
| default: '4096' | |
| request_timeout_seconds: | |
| description: Maximum seconds per Ollama request | |
| required: false | |
| default: '600' | |
| push: | |
| branches: [agent/phase9-empirical-consumer] | |
| pull_request: | |
| paths: | |
| - 'tools/mixed_register_empirical.py' | |
| - 'tools/mixed_register_empirical_closure.py' | |
| - 'tools/run_mixed_register_empirical.py' | |
| - 'tools/close_mixed_register_empirical.py' | |
| - 'tools/mode_core.py' | |
| - 'tools/retrieve_mode_context.py' | |
| - 'ai/manifest.json' | |
| - 'schema/manifest.schema.json' | |
| - 'tests/test_phase9_empirical.py' | |
| - 'tests/test_phase9_empirical_closure.py' | |
| - 'schema/mixed-register-empirical-closure.schema.json' | |
| - 'empirical/mixed-register/**' | |
| - 'docs/MIXED_REGISTER_EMPIRICAL.md' | |
| - '.github/workflows/phase9-empirical-consumer.yml' | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: phase9-empirical-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| static: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install validation dependencies | |
| run: python -m pip install -r requirements-validation.txt | |
| - name: Compile empirical runner | |
| run: | | |
| python -m py_compile tools/mixed_register_empirical.py | |
| python -m py_compile tools/mixed_register_empirical_closure.py | |
| python -m py_compile tools/run_mixed_register_empirical.py | |
| python -m py_compile tools/close_mixed_register_empirical.py | |
| - name: Run empirical regression tests | |
| run: | | |
| python -m unittest tests.test_phase9_empirical | |
| python -m unittest tests.test_phase9_empirical_closure | |
| - name: Run Phase 9 regression suite | |
| run: python -m unittest discover -s tests -p 'test_phase9*.py' | |
| empirical: | |
| # The ten-pass Ollama matrix remains manual by default. PR #17 is the one | |
| # deliberate exception, and only when the head branch belongs to this repo. | |
| if: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.number == 17 && github.event.pull_request.head.repo.full_name == github.repository && github.head_ref == 'agent/phase9-cold-consumer-closure') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| env: | |
| MODEL: ${{ inputs.model || 'qwen2.5:3b' }} | |
| NUM_CTX: ${{ inputs.num_ctx || '32768' }} | |
| # The CPU-only PR #17 runner timed out at the 4,096-token ceiling on its | |
| # first MICRO pass. Bound only this one PR to 2,048 generated tokens; | |
| # workflow_dispatch and the machine-readable protocol keep 4,096. | |
| NUM_PREDICT: ${{ github.event_name == 'pull_request' && github.event.pull_request.number == 17 && '2048' || inputs.num_predict || '4096' }} | |
| REQUEST_TIMEOUT_SECONDS: ${{ inputs.request_timeout_seconds || '600' }} | |
| PYTHONUNBUFFERED: '1' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install validation dependencies | |
| run: python -m pip install -r requirements-validation.txt | |
| - name: Install and start Ollama | |
| run: | | |
| curl -fsSL https://ollama.com/install.sh | sh | |
| nohup ollama serve >/tmp/ollama.log 2>&1 & | |
| for i in $(seq 1 60); do | |
| if curl -fsS http://127.0.0.1:11434/api/tags >/dev/null; then | |
| exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| cat /tmp/ollama.log | |
| exit 1 | |
| - name: Pull local cold-consumer model | |
| run: ollama pull "$MODEL" | |
| - name: Run paired guarded and ablated experiment | |
| run: | | |
| python -u tools/run_mixed_register_empirical.py \ | |
| --model "$MODEL" \ | |
| --num-ctx "$NUM_CTX" \ | |
| --num-predict "$NUM_PREDICT" \ | |
| --request-timeout-seconds "$REQUEST_TIMEOUT_SECONDS" \ | |
| --source-commit "$(git rev-parse HEAD)" \ | |
| --output dist/empirical/mixed-register | |
| - name: Close Phase 9 empirical questions | |
| run: | | |
| python -u tools/close_mixed_register_empirical.py \ | |
| --empirical-dir dist/empirical/mixed-register \ | |
| --output dist/empirical/mixed-register/closure.json \ | |
| --markdown dist/empirical/mixed-register/closure.md | |
| - name: Require strict cold-consumer closure | |
| run: | | |
| python - <<'PY' | |
| import json | |
| from pathlib import Path | |
| closure = json.loads(Path('dist/empirical/mixed-register/closure.json').read_text(encoding='utf-8')) | |
| if not closure['cold_consumer_classification_demonstrated']: | |
| raise SystemExit( | |
| 'Phase 9 cold-consumer exit criterion NOT demonstrated; ' | |
| 'ROADMAP.md must remain unchecked for this immutable run.' | |
| ) | |
| print('Phase 9 cold-consumer exit criterion demonstrated.') | |
| print('strict_passing_guarded_conditions=' + ','.join(closure['strict_passing_guarded_conditions'])) | |
| PY | |
| - name: Publish empirical summary | |
| if: always() | |
| run: | | |
| python - <<'PY' | |
| import json, os | |
| from pathlib import Path | |
| summary_path = Path('dist/empirical/mixed-register/summary.json') | |
| closure_path = Path('dist/empirical/mixed-register/closure.json') | |
| lines = ['## Phase 9 empirical mixed-register run', ''] | |
| if summary_path.is_file(): | |
| data = json.loads(summary_path.read_text(encoding='utf-8')) | |
| lines.extend([ | |
| f"Model: `{data['model']['model_id']}`", | |
| f"Immutable revision: `{data['model']['immutable_model_revision']}`", | |
| f"Original cold-consumer gate demonstrated: **{data['cold_consumer_demonstrated']}**", | |
| f"Passing guarded conditions: `{', '.join(data['passing_guarded_conditions']) or 'none'}`", | |
| '', | |
| ]) | |
| if closure_path.is_file(): | |
| closure = json.loads(closure_path.read_text(encoding='utf-8')) | |
| lines.extend([ | |
| f"Adjacency-trap claims: `{closure['adjacency_trap_claim_count']}`", | |
| f"Guard-effect conclusion: **{closure['guard_effect_conclusion']}**", | |
| f"Local guards improved in any condition: **{closure['local_guards_improved_in_any_condition']}**", | |
| f"Consistent non-degradation with improvement: **{closure['local_guards_consistent_non_degradation']}**", | |
| f"Strict cold-consumer classification demonstrated: **{closure['cold_consumer_classification_demonstrated']}**", | |
| f"Strict passing guarded conditions: `{', '.join(closure['strict_passing_guarded_conditions']) or 'none'}`", | |
| '', | |
| '| Condition | Effect | Adj. status Δ | False-support reduction | Unavailable Δ | Strict cold gate |', | |
| '|---|---|---:|---:|---:|---:|', | |
| ]) | |
| for row in closure['rows']: | |
| effect = row['guard_effect'] | |
| lines.append( | |
| f"| {row['condition']} | {row['guard_effect_classification']} | " | |
| f"{effect['adjacency_status_accuracy_delta']} | " | |
| f"{effect['adjacency_false_support_rate_reduction']} | " | |
| f"{effect['unavailable_status_accuracy_delta']} | " | |
| f"{row['strict_cold_consumer_gate']['passed']} |" | |
| ) | |
| else: | |
| lines.append('Empirical closure artifact was not produced.') | |
| with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a', encoding='utf-8') as handle: | |
| handle.write('\n'.join(lines) + '\n') | |
| PY | |
| - name: Upload empirical evidence | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: phase9-mixed-register-empirical-${{ github.run_id }} | |
| path: dist/empirical/mixed-register | |
| if-no-files-found: warn | |
| retention-days: 30 |