Skip to content

Commit 9759792

Browse files
superpiossuperpios
authored
Add files via upload
1 parent b633410 commit 9759792

1 file changed

Lines changed: 248 additions & 0 deletions

File tree

‎pipeline.yml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
# DVNS Investigative Pipeline — end-to-end
2+
# Explorer relations → Leads Generator → Alert Engine ranking
3+
#
4+
# Public repo = unlimited free GitHub Actions minutes.
5+
# Schedule: daily 03:00 UTC + manual workflow_dispatch.
6+
#
7+
# Philosophy (binding):
8+
# Every output is a quantitative signal that merits human verification.
9+
# No automatic conclusion of wrongdoing, waste, fraud or individual liability.
10+
11+
name: DVNS Investigative Pipeline
12+
13+
on:
14+
schedule:
15+
# Every day at 03:00 UTC (adjust as needed)
16+
- cron: "0 3 * * *"
17+
workflow_dispatch:
18+
inputs:
19+
use_fixture_data:
20+
description: "Use synthetic fixture data (for smoke test) instead of live Explorer relations"
21+
required: false
22+
default: "false"
23+
type: choice
24+
options:
25+
- "false"
26+
- "true"
27+
skip_publish:
28+
description: "Skip committing ranked feed to data branch"
29+
required: false
30+
default: "false"
31+
type: choice
32+
options:
33+
- "false"
34+
- "true"
35+
36+
# Prevent concurrent runs from overlapping
37+
concurrency:
38+
group: dvns-pipeline
39+
cancel-in-progress: false
40+
41+
permissions:
42+
contents: write # needed only if publishing to data branch / gh-pages
43+
44+
env:
45+
PYTHON_VERSION: "3.11"
46+
# Pin clones to main; change to a tag/SHA for stricter reproducibility
47+
EXPLORER_REF: main
48+
LEADS_REF: main
49+
ALERT_REF: main
50+
51+
jobs:
52+
pipeline:
53+
name: Run full investigative cycle
54+
runs-on: ubuntu-latest
55+
timeout-minutes: 45
56+
57+
steps:
58+
- name: Checkout coordinator (this repo)
59+
uses: actions/checkout@v4
60+
with:
61+
path: coordinator
62+
63+
- name: Set up Python
64+
uses: actions/setup-python@v5
65+
with:
66+
python-version: ${{ env.PYTHON_VERSION }}
67+
68+
- name: Install shared dependencies
69+
run: |
70+
python -m pip install --upgrade pip
71+
pip install "pyyaml>=6.0" "pandas>=2.0"
72+
73+
- name: Clone investigative-explorer-dvns
74+
if: ${{ github.event.inputs.use_fixture_data != 'true' }}
75+
run: |
76+
git clone --depth 1 --branch "${{ env.EXPLORER_REF }}" \
77+
https://github.com/superpios/investigative-explorer-dvns.git explorer
78+
echo "Explorer relations present:"
79+
ls -la explorer/data/relations/ | head -30
80+
81+
- name: Clone investigative-leads-generator
82+
run: |
83+
git clone --depth 1 --branch "${{ env.LEADS_REF }}" \
84+
https://github.com/superpios/investigative-leads-generator.git leads
85+
86+
- name: Clone investigative-alert-engine
87+
run: |
88+
git clone --depth 1 --branch "${{ env.ALERT_REF }}" \
89+
https://github.com/superpios/investigative-alert-engine.git alert
90+
91+
- name: Prepare workspace
92+
run: |
93+
mkdir -p work/{input,leads,ranked,history}
94+
# Carry over previous history if published in this repo (optional)
95+
if [ -d coordinator/data/history ]; then
96+
cp -a coordinator/data/history/. work/history/ || true
97+
fi
98+
- name: Select relations source
99+
id: relations
100+
run: |
101+
if [ "${{ github.event.inputs.use_fixture_data }}" = "true" ]; then
102+
REL="coordinator/testdata/relations"
103+
echo "Using fixture data: $REL"
104+
else
105+
REL="explorer/data/relations"
106+
echo "Using live Explorer relations: $REL"
107+
fi
108+
missing=0
109+
for f in persona_incarico_ente__incarichi_nominativi_shard.csv \
110+
awards__affidamenti_diretti.csv \
111+
cig_ente__affidamenti_diretti.csv; do
112+
if [ ! -f "$REL/$f" ]; then
113+
echo "::error::Missing required relation file: $REL/$f"
114+
missing=1
115+
fi
116+
done
117+
if [ "$missing" != 0 ]; then
118+
echo "::error::Fail-closed: required Explorer relation CSVs are missing."
119+
exit 1
120+
fi
121+
echo "path=$REL" >> "$GITHUB_OUTPUT"
122+
123+
- name: Step 1 — Adapt Explorer relations → generator input
124+
run: |
125+
python leads/scripts/adapt_explorer.py \
126+
--relations "${{ steps.relations.outputs.path }}" \
127+
--output work/input
128+
echo "Adapted input files:"
129+
ls -la work/input/
130+
wc -l work/input/*.csv || true
131+
132+
- name: Step 2 — Generate conservative leads
133+
run: |
134+
set +e
135+
python leads/scripts/apply_rules.py \
136+
--input work/input \
137+
--output work/leads \
138+
--rules leads/rules/rules_v0.1.yaml
139+
rc=$?
140+
set -e
141+
echo "apply_rules exit code: $rc"
142+
if [ -f work/leads/manifest.json ]; then
143+
echo "Manifest:"
144+
cat work/leads/manifest.json
145+
fi
146+
# Fail-closed on broken input (exit 1) is intentional and should fail the job
147+
if [ $rc -ne 0 ]; then
148+
echo "::error::Leads generator failed (fail-closed on invalid input). See manifest."
149+
exit $rc
150+
fi
151+
# Zero leads is a valid conservative outcome (exit 0)
152+
ls -la work/leads/ || true
153+
154+
- name: Step 3 — Rank leads + update history
155+
run: |
156+
# rank_leads exits 0 even with zero leads (fail-closed content)
157+
python alert/scripts/rank_leads.py \
158+
--input work/leads \
159+
--output work/ranked \
160+
--history work/history \
161+
--rules alert/rules/ranking_v0.1.yaml \
162+
--entity-config alert/config/entity_names.yaml
163+
ls -la work/ranked/ work/history/ || true
164+
if [ -f work/ranked/ranked_leads.json ]; then
165+
python -c "
166+
import json
167+
r = json.load(open('work/ranked/ranked_leads.json'))
168+
print(f'Ranked leads: {len(r)}')
169+
for L in r[:10]:
170+
print(f\" #{L.get('rank_position')} score={L.get('priority_score')} {L.get('id')} | {L.get('title','')[:70]}\")
171+
"
172+
else
173+
echo "No ranked_leads.json (zero leads or fail-closed). OK."
174+
fi
175+
176+
- name: Step 4 — Export human-readable feed (optional)
177+
run: |
178+
if [ -f work/ranked/ranked_leads.json ]; then
179+
python alert/scripts/export_feed.py \
180+
--input work/ranked \
181+
--output work/ranked/feed.md || echo "export_feed skipped/failed (non-fatal)"
182+
fi
183+
ls -la work/ranked/ || true
184+
185+
- name: Audit checks (schema + disclaimer + determinism marker)
186+
run: |
187+
python coordinator/scripts/audit_output.py \
188+
--ranked work/ranked/ranked_leads.json \
189+
--manifest work/leads/manifest.json \
190+
--history work/history || true
191+
# Soft audit: do not fail the whole job on zero leads
192+
193+
- name: Upload artifacts
194+
uses: actions/upload-artifact@v4
195+
with:
196+
name: dvns-ranked-${{ github.run_id }}
197+
path: |
198+
work/ranked/
199+
work/history/
200+
work/leads/manifest.json
201+
work/leads/leads_v0.1.json
202+
retention-days: 30
203+
if-no-files-found: warn
204+
205+
- name: Publish ranked feed to data branch
206+
if: ${{ github.event.inputs.skip_publish != 'true' && github.ref == 'refs/heads/main' }}
207+
run: |
208+
if [ ! -f work/ranked/ranked_leads.json ]; then
209+
echo "Nothing to publish."
210+
exit 0
211+
fi
212+
# Commit only the public outputs into a dedicated branch for consumption by the website
213+
git config --global user.name "dvns-pipeline-bot"
214+
git config --global user.email "pipeline@users.noreply.github.com"
215+
# Use a worktree-like approach on the same checkout
216+
mkdir -p coordinator/data/ranked coordinator/data/history
217+
cp -a work/ranked/. coordinator/data/ranked/
218+
cp -a work/history/. coordinator/data/history/
219+
cd coordinator
220+
git add data/ranked data/history
221+
if git diff --staged --quiet; then
222+
echo "No changes to publish."
223+
else
224+
git commit -m "chore(pipeline): ranked feed $(date -u +%Y-%m-%dT%H:%MZ) [skip ci]"
225+
git push origin HEAD:main || git push origin HEAD:data || true
226+
fi
227+
228+
- name: Summary
229+
if: always()
230+
run: |
231+
echo "## DVNS Pipeline summary" >> "$GITHUB_STEP_SUMMARY"
232+
echo "" >> "$GITHUB_STEP_SUMMARY"
233+
if [ -f work/leads/manifest.json ]; then
234+
echo '```json' >> "$GITHUB_STEP_SUMMARY"
235+
cat work/leads/manifest.json >> "$GITHUB_STEP_SUMMARY"
236+
echo '```' >> "$GITHUB_STEP_SUMMARY"
237+
fi
238+
if [ -f work/ranked/ranked_leads.json ]; then
239+
N=$(python -c "import json; print(len(json.load(open('work/ranked/ranked_leads.json'))))")
240+
echo "" >> "$GITHUB_STEP_SUMMARY"
241+
echo "**Ranked leads:** $N" >> "$GITHUB_STEP_SUMMARY"
242+
echo "" >> "$GITHUB_STEP_SUMMARY"
243+
echo "Artifact: \`dvns-ranked-${{ github.run_id }}\`" >> "$GITHUB_STEP_SUMMARY"
244+
else
245+
echo "No ranked output (zero leads or fail-closed)." >> "$GITHUB_STEP_SUMMARY"
246+
fi
247+
echo "" >> "$GITHUB_STEP_SUMMARY"
248+
echo "> Ogni pista è un segnale quantitativo che merita verifica umana. Nessuna conclusione automatica di illecito." >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)