Update DATA_CLEANING_AND_CODING.md #146
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Public Release CI | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| test: | |
| name: Public release checks (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| PYTHONUTF8: "1" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Compile Python files | |
| run: python -m compileall demo/src pipeline tools tests demo/tests | |
| - name: Ruff - Demo | |
| run: ruff check demo/src demo/tests tools/run_demo.py | |
| - name: Ruff - Pipeline | |
| run: ruff check pipeline | |
| - name: Ruff - Public normalization | |
| run: ruff check tools/normalize_public_dataset.py | |
| - name: Ruff - Public analysis | |
| run: ruff check tools/summarize_public_analysis.py | |
| - name: Ruff - Project brief | |
| run: ruff check tools/build_project_brief_docx.py | |
| - name: Ruff - Other tools | |
| shell: bash | |
| run: | | |
| files=$(find tools -maxdepth 1 -name '*.py' \ | |
| ! -name 'run_demo.py' \ | |
| ! -name 'normalize_public_dataset.py' \ | |
| ! -name 'summarize_public_analysis.py' \ | |
| ! -name 'build_project_brief_docx.py' -print) | |
| if [ -n "$files" ]; then ruff check $files; fi | |
| - name: Ruff - Tests | |
| run: ruff check tests | |
| - name: Ruff - Full tree | |
| run: ruff check demo/src demo/tests pipeline tools tests | |
| - name: Public copy P1 gate | |
| run: >- | |
| python tools/lint_public_copy.py --root . --quality-gate P1 | |
| --format json --output "${{ runner.temp }}/public_copy_report.json" | |
| - name: Test suite | |
| run: python -m pytest demo/tests tests -q | |
| - name: Calibration pipeline smoke (mock, temp dir) | |
| shell: bash | |
| run: | | |
| python tools/calibration/build_calibration_sample.py --config config/calibration/calibration.yaml | |
| python tools/calibration/run_agent_reviews.py --provider mock \ | |
| --input data/calibration/calibration_sample.csv --max-items 30 \ | |
| --output "$RUNNER_TEMP/cal_reviews" | |
| python tools/calibration/build_agent_consensus.py \ | |
| --input-dir "$RUNNER_TEMP/cal_reviews" --output-dir "$RUNNER_TEMP/cal_consensus" | |
| python tools/calibration/analyze_disagreements.py \ | |
| --input "$RUNNER_TEMP/cal_consensus/consensus_reference.csv" \ | |
| --output "$RUNNER_TEMP/cal_consensus/disagreement_report.md" \ | |
| --codebook-output "$RUNNER_TEMP/cal_consensus/codebook_revision_proposals.md" | |
| - name: Verify calibration outputs and public safety | |
| shell: bash | |
| env: | |
| RUN_REPORT: ${{ runner.temp }}/cal_reviews/run_report.json | |
| CONS_REPORT: ${{ runner.temp }}/cal_consensus/calibration_report.json | |
| run: | | |
| python tools/calibration/verify_mock_outputs.py \ | |
| --run-report "$RUN_REPORT" \ | |
| --consensus-report "$CONS_REPORT" \ | |
| --public-sample data/calibration/calibration_sample.csv | |
| git diff --quiet -- data/public docs/files | |
| - name: Normalize public data smoke | |
| run: python tools/normalize_public_dataset.py --source-dir data/public --output-dir artifacts/normalized_public | |
| - name: Analysis summary smoke | |
| run: python tools/summarize_public_analysis.py --public-dir data/public --output artifacts/public_analysis_summary.json | |
| - name: Project brief smoke | |
| run: python tools/build_project_brief_docx.py --output artifacts/PsyLens_project_brief.docx | |
| - name: Demo smoke | |
| run: python tools/run_demo.py --provider mock --output artifacts/demo/ci | |
| - name: Upload QA artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: psylens-ci-${{ matrix.os }} | |
| path: | | |
| artifacts/public_analysis_summary.json | |
| artifacts/PsyLens_project_brief.docx | |
| artifacts/demo/ci | |
| if-no-files-found: error |