|
| 1 | +#!/usr/bin/env python3 |
| 2 | +import argparse, csv, hashlib, json |
| 3 | +from pathlib import Path |
| 4 | +ROOT=Path(__file__).resolve().parents[1] |
| 5 | +def dump(p,v): p.parent.mkdir(parents=True,exist_ok=True); p.write_text(json.dumps(v,indent=2)+"\n") |
| 6 | +def ingest(out): |
| 7 | + docs=[] |
| 8 | + for p in sorted((ROOT/'data/documents').glob('*.txt')): |
| 9 | + text=p.read_text(); docs.append({'file':p.name,'sha256':hashlib.sha256(text.encode()).hexdigest(),'text':text}) |
| 10 | + dump(out/'ingested.json',docs) |
| 11 | +def classify(out): |
| 12 | + docs=json.loads((out/'ingested.json').read_text()) |
| 13 | + dump(out/'classified.json',[{**d,'type':next(x.split(':',1)[1].strip().lower() for x in d['text'].splitlines() if x.startswith('DOCUMENT:'))} for d in docs]) |
| 14 | +def extract(out): |
| 15 | + result=[] |
| 16 | + for d in json.loads((out/'classified.json').read_text()): |
| 17 | + fields={}; items=[] |
| 18 | + for line in d['text'].splitlines(): |
| 19 | + if ': ' not in line: continue |
| 20 | + key,val=line.split(': ',1) |
| 21 | + if key=='ITEM': |
| 22 | + name,qty,price=[x.strip() for x in val.split('|')]; items.append({'name':name,'quantity':int(qty),'unit_price':float(price)}) |
| 23 | + else: fields[key.lower()]=val |
| 24 | + result.append({'file':d['file'],'sha256':d['sha256'],'type':d['type'],'fields':fields,'items':items}) |
| 25 | + dump(out/'extracted.json',result) |
| 26 | +def reconcile(out): |
| 27 | + docs=json.loads((out/'extracted.json').read_text()); rows=[]; seen=set() |
| 28 | + for d in docs: |
| 29 | + if d['type']!='invoice': continue |
| 30 | + f=d['fields']; subtotal=sum(i['quantity']*i['unit_price'] for i in d['items']); total=float(f['total']); tax=float(f['tax']); iid=f['invoice_id'] |
| 31 | + rows.append({'invoice_id':iid,'supplier':f['supplier'],'subtotal':subtotal,'tax':tax,'total':total,'balanced':abs(subtotal+tax-total)<.01,'duplicate':iid in seen}); seen.add(iid) |
| 32 | + dump(out/'reconciliation.json',rows) |
| 33 | + with (out/'invoice-summary.csv').open('w',newline='') as fp: |
| 34 | + w=csv.DictWriter(fp,fieldnames=list(rows[0])); w.writeheader(); w.writerows(rows) |
| 35 | +def report(out): |
| 36 | + rows=json.loads((out/'reconciliation.json').read_text()); ok=all(r['balanced'] and not r['duplicate'] for r in rows) |
| 37 | + dump(out/'report.json',{'status':'passed' if ok else 'failed','documents':len(json.loads((out/'ingested.json').read_text())),'invoices':len(rows),'gross_total':sum(r['total'] for r in rows)}) |
| 38 | + artifacts=['ingested.json','classified.json','extracted.json','reconciliation.json','invoice-summary.csv','report.json'] |
| 39 | + dump(out/'manifest.json',{'showcase':'document-intelligence','status':'success' if ok else 'failed','artifacts':artifacts}) |
| 40 | +STAGES={'ingest':ingest,'classify':classify,'extract':extract,'reconcile':reconcile,'report':report} |
| 41 | +def main(): |
| 42 | + p=argparse.ArgumentParser();p.add_argument('--stage',choices=[*STAGES,'all'],default='all');p.add_argument('--output',required=True);a=p.parse_args();o=Path(a.output);o.mkdir(parents=True,exist_ok=True) |
| 43 | + for s in list(STAGES) if a.stage=='all' else [a.stage]: STAGES[s](o) |
| 44 | +if __name__=='__main__':main() |
| 45 | + |
0 commit comments