Improve error handling for missing relation files #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # DVNS Investigative Pipeline — end-to-end | ||
| # Explorer relations → Leads Generator → Alert Engine ranking | ||
| # | ||
| # Public repo = unlimited free GitHub Actions minutes. | ||
| # Schedule: daily 03:00 UTC + manual workflow_dispatch. | ||
| # | ||
| # Philosophy (binding): | ||
| # Every output is a quantitative signal that merits human verification. | ||
| # No automatic conclusion of wrongdoing, waste, fraud or individual liability. | ||
| name: DVNS Investigative Pipeline | ||
| on: | ||
| schedule: | ||
| # Every day at 03:00 UTC (adjust as needed) | ||
| - cron: "0 3 * * *" | ||
| workflow_dispatch: | ||
| inputs: | ||
| use_fixture_data: | ||
| description: "Use synthetic fixture data (for smoke test) instead of live Explorer relations" | ||
| required: false | ||
| default: "false" | ||
| type: choice | ||
| options: | ||
| - "false" | ||
| - "true" | ||
| skip_publish: | ||
| description: "Skip committing ranked feed to data branch" | ||
| required: false | ||
| default: "false" | ||
| type: choice | ||
| options: | ||
| - "false" | ||
| - "true" | ||
| # Prevent concurrent runs from overlapping | ||
| concurrency: | ||
| group: dvns-pipeline | ||
| cancel-in-progress: false | ||
| permissions: | ||
| contents: write # needed only if publishing to data branch / gh-pages | ||
| env: | ||
| PYTHON_VERSION: "3.11" | ||
| # Pin clones to main; change to a tag/SHA for stricter reproducibility | ||
| EXPLORER_REF: main | ||
| LEADS_REF: main | ||
| ALERT_REF: main | ||
| jobs: | ||
| pipeline: | ||
| name: Run full investigative cycle | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 45 | ||
| steps: | ||
| - name: Checkout coordinator (this repo) | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: coordinator | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ env.PYTHON_VERSION }} | ||
| - name: Install shared dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install "pyyaml>=6.0" "pandas>=2.0" | ||
| - name: Clone investigative-explorer-dvns | ||
| if: ${{ github.event.inputs.use_fixture_data != 'true' }} | ||
| run: | | ||
| git clone --depth 1 --branch "${{ env.EXPLORER_REF }}" \ | ||
| https://github.com/superpios/investigative-explorer-dvns.git explorer | ||
| echo "Explorer relations present:" | ||
| ls -la explorer/data/relations/ | head -30 | ||
| - name: Clone investigative-leads-generator | ||
| run: | | ||
| git clone --depth 1 --branch "${{ env.LEADS_REF }}" \ | ||
| https://github.com/superpios/investigative-leads-generator.git leads | ||
| - name: Clone investigative-alert-engine | ||
| run: | | ||
| git clone --depth 1 --branch "${{ env.ALERT_REF }}" \ | ||
| https://github.com/superpios/investigative-alert-engine.git alert | ||
| - name: Prepare workspace | ||
| run: | | ||
| mkdir -p work/{input,leads,ranked,history} | ||
| # Carry over previous history if published in this repo (optional) | ||
| if [ -d coordinator/data/history ]; then | ||
| cp -a coordinator/data/history/. work/history/ || true | ||
| fi | ||
| - name: Select relations source | ||
| id: relations | ||
| run: | | ||
| if [ "${{ github.event.inputs.use_fixture_data }}" = "true" ]; then | ||
| REL="coordinator/testdata/relations" | ||
| echo "Using fixture data: $REL" | ||
| else | ||
| REL="explorer/data/relations" | ||
| echo "Using live Explorer relations: $REL" | ||
| fi | ||
| missing=0for f in persona_incarico_ente__incarichi_nominativi_shard.csv \ | ||
| awards__affidamenti_diretti.csv \ | ||
| cig_ente__affidamenti_diretti.csv; do | ||
| if [ ! -f "$REL/$f" ]; then | ||
| echo "::error::Missing required relation file: $REL/$f" | ||
| missing=1 | ||
| fi | ||
| done | ||
| if [ "$missing" != 0 ]; then | ||
| echo "::error::Fail-closed: required Explorer relation CSVs are missing." | ||
| exit 1 | ||
| fi | ||
| echo "path=$REL" >> "$GITHUB_OUTPUT" | ||
| - name: Step 1 — Adapt Explorer relations → generator input | ||
| run: | | ||
| python leads/scripts/adapt_explorer.py \ | ||
| --relations "${{ steps.relations.outputs.path }}" \ | ||
| --output work/input | ||
| echo "Adapted input files:" | ||
| ls -la work/input/ | ||
| wc -l work/input/*.csv || true | ||
| - name: Step 2 — Generate conservative leads | ||
| run: | | ||
| set +e | ||
| python leads/scripts/apply_rules.py \ | ||
| --input work/input \ | ||
| --output work/leads \ | ||
| --rules leads/rules/rules_v0.1.yaml | ||
| rc=$? | ||
| set -e | ||
| echo "apply_rules exit code: $rc" | ||
| if [ -f work/leads/manifest.json ]; then | ||
| echo "Manifest:" | ||
| cat work/leads/manifest.json | ||
| fi | ||
| # Fail-closed on broken input (exit 1) is intentional and should fail the job | ||
| if [ $rc -ne 0 ]; then | ||
| echo "::error::Leads generator failed (fail-closed on invalid input). See manifest." | ||
| exit $rc | ||
| fi | ||
| # Zero leads is a valid conservative outcome (exit 0) | ||
| ls -la work/leads/ || true | ||
| - name: Step 3 — Rank leads + update history | ||
| run: | | ||
| # rank_leads exits 0 even with zero leads (fail-closed content) | ||
| python alert/scripts/rank_leads.py \ | ||
| --input work/leads \ | ||
| --output work/ranked \ | ||
| --history work/history \ | ||
| --rules alert/rules/ranking_v0.1.yaml \ | ||
| --entity-config alert/config/entity_names.yaml | ||
| ls -la work/ranked/ work/history/ || true | ||
| if [ -f work/ranked/ranked_leads.json ]; then | ||
| python -c " | ||
| import json | ||
| r = json.load(open('work/ranked/ranked_leads.json')) | ||
| print(f'Ranked leads: {len(r)}') | ||
| for L in r[:10]: | ||
| print(f\" #{L.get('rank_position')} score={L.get('priority_score')} {L.get('id')} | {L.get('title','')[:70]}\") | ||
| " | ||
| else | ||
| echo "No ranked_leads.json (zero leads or fail-closed). OK." | ||
| fi | ||
| - name: Step 4 — Export human-readable feed (optional) | ||
| run: | | ||
| if [ -f work/ranked/ranked_leads.json ]; then | ||
| python alert/scripts/export_feed.py \ | ||
| --input work/ranked \ | ||
| --output work/ranked/feed.md || echo "export_feed skipped/failed (non-fatal)" | ||
| fi | ||
| ls -la work/ranked/ || true | ||
| - name: Audit checks (schema + disclaimer + determinism marker) | ||
| run: | | ||
| python coordinator/scripts/audit_output.py \ | ||
| --ranked work/ranked/ranked_leads.json \ | ||
| --manifest work/leads/manifest.json \ | ||
| --history work/history || true | ||
| # Soft audit: do not fail the whole job on zero leads | ||
| - name: Upload artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: dvns-ranked-${{ github.run_id }} | ||
| path: | | ||
| work/ranked/ | ||
| work/history/ | ||
| work/leads/manifest.json | ||
| work/leads/leads_v0.1.json | ||
| retention-days: 30 | ||
| if-no-files-found: warn | ||
| - name: Publish ranked feed to data branch | ||
| if: ${{ github.event.inputs.skip_publish != 'true' && github.ref == 'refs/heads/main' }} | ||
| run: | | ||
| if [ ! -f work/ranked/ranked_leads.json ]; then | ||
| echo "Nothing to publish." | ||
| exit 0 | ||
| fi | ||
| # Commit only the public outputs into a dedicated branch for consumption by the website | ||
| git config --global user.name "dvns-pipeline-bot" | ||
| git config --global user.email "pipeline@users.noreply.github.com" | ||
| # Use a worktree-like approach on the same checkout | ||
| mkdir -p coordinator/data/ranked coordinator/data/history | ||
| cp -a work/ranked/. coordinator/data/ranked/ | ||
| cp -a work/history/. coordinator/data/history/ | ||
| cd coordinator | ||
| git add data/ranked data/history | ||
| if git diff --staged --quiet; then | ||
| echo "No changes to publish." | ||
| else | ||
| git commit -m "chore(pipeline): ranked feed $(date -u +%Y-%m-%dT%H:%MZ) [skip ci]" | ||
| git push origin HEAD:main || git push origin HEAD:data || true | ||
| fi | ||
| - name: Summary | ||
| if: always() | ||
| run: | | ||
| echo "## DVNS Pipeline summary" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "" >> "$GITHUB_STEP_SUMMARY" | ||
| if [ -f work/leads/manifest.json ]; then | ||
| echo '```json' >> "$GITHUB_STEP_SUMMARY" | ||
| cat work/leads/manifest.json >> "$GITHUB_STEP_SUMMARY" | ||
| echo '```' >> "$GITHUB_STEP_SUMMARY" | ||
| fi | ||
| if [ -f work/ranked/ranked_leads.json ]; then | ||
| N=$(python -c "import json; print(len(json.load(open('work/ranked/ranked_leads.json'))))") | ||
| echo "" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "**Ranked leads:** $N" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "Artifact: \`dvns-ranked-${{ github.run_id }}\`" >> "$GITHUB_STEP_SUMMARY" | ||
| else | ||
| echo "No ranked output (zero leads or fail-closed)." >> "$GITHUB_STEP_SUMMARY" | ||
| fi | ||
| echo "" >> "$GITHUB_STEP_SUMMARY" | ||
| echo "> Ogni pista è un segnale quantitativo che merita verifica umana. Nessuna conclusione automatica di illecito." >> "$GITHUB_STEP_SUMMARY" | ||
